MetadataManager.py 54.3 KB
Newer Older
Christophe Cleva's avatar
Christophe Cleva committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
#!/usr/bin/env python
# -*- coding:utf-8 -*- 


##############################################################################
## license :
##============================================================================
##
## File :        MetadataManager.py
## 
## Project :     Metadata Manager for ICAT
##
## This file is part of Tango device class.
## 
## Tango is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
## 
## Tango is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
## 
## You should have received a copy of the GNU General Public License
## along with Tango.  If not, see <http://www.gnu.org/licenses/>.
## 
##
## $Author :      christophe.cleva$
##
## $Revision :    $
##
## $Date :        $
##
## $HeadUrl :     $
##============================================================================
##            This file is generated by POGO
##    (Program Obviously used to Generate tango Object)
##############################################################################

__all__ = ["MetadataManager", "MetadataManagerClass", "main"]

__docformat__ = 'restructuredtext'

45
import tango
Christophe Cleva's avatar
Christophe Cleva committed
46 47 48
import sys
# Add additional import
#----- PROTECTED REGION ID(MetadataManager.additionnal_import) ENABLED START -----#
49
import os.path
50
import sys, traceback
51
import string
52
import json
53
from datetime import datetime
54
from time import gmtime, strftime
Christophe Cleva's avatar
Christophe Cleva committed
55
import logging
56 57 58 59
try:
    import requests
except ImportError:
    requests = None
60 61 62 63 64
try:
    import graypy
except ImportError:
    graypy = None
    
65
from collections import deque
66

Matias Guijarro's avatar
Matias Guijarro committed
67 68
from metadata_manager.icat_ingest_metadata.tomodb import tomodbFile
from metadata_manager.icat_ingest_metadata.dataset import dataset, parameter, sample, datafile
Christophe Cleva's avatar
Christophe Cleva committed
69

70
from metadata_manager.MessagingClient import StompClient
71

72 73
from metadata_manager.TangoLoggingHandler import TangoLoggingHandler
from metadata_manager.ParameterDefinition import ParameterDefinition
74

75 76
from metadata_manager.ICATParameterReader import ICATParameterReader
from metadata_manager.MetadataWorker import MetadataWorker
77 78
from metadata_manager import tangoworker

Christophe Cleva's avatar
Christophe Cleva committed
79

Christophe Cleva's avatar
Christophe Cleva committed
80 81 82
#----- PROTECTED REGION END -----#	//	MetadataManager.additionnal_import

## Device States Description
83
## FAULT : Device is not functioning correctly
Matias Guijarro's avatar
Matias Guijarro committed
84
## ON : Device is ready to record dataset
85
## OFF : No experiment ongoing
Matias Guijarro's avatar
Matias Guijarro committed
86 87
## STANDBY : Experiment started, sample or dataset name is missing
## RUNNING : Dataset currently recorded
Christophe Cleva's avatar
Christophe Cleva committed
88

89

90
class MetadataManager (tango.LatestDeviceImpl):
91 92
    """"""
    def __init__(self, cl, name):
93
        tango.LatestDeviceImpl.__init__(self,cl,name)
94
        self.metadataWorker = MetadataWorker(self.get_name())
95
        MetadataManager.init_device(self)
Christophe Cleva's avatar
Christophe Cleva committed
96 97
        
    def delete_device(self):
98
        self.debug_stream("In delete_device()")
Christophe Cleva's avatar
Christophe Cleva committed
99
        #----- PROTECTED REGION ID(MetadataManager.delete_device) ENABLED START -----#
100 101 102 103 104 105 106 107
        try:
            if self.expdev.is_locked_by_me():
                try:
                    self.expdev.unlock()
                except Exception as e:
                    self.error_stream("Error unlocking MetaExp: %s" % e)
        except Exception as e:
            self.warn_stream("Error accessing MetaExp: %s" % e)
108 109 110 111 112 113 114 115 116 117 118 119 120 121
        try:
            self.expdev.unsubscribe_event(self.proposalEventId)
        except Exception as e:
            self.error_stream("Error unsubscribing: %s" % e)
        try:
            self.expdev.unsubscribe_event(self.sampleEventId)
        except Exception as e:
            self.error_stream("Error unsubscribing: %s" % e)
        try:
            self.expdev.unsubscribe_event(self.dataRootEventId)
        except Exception as e:
            self.error_stream("Error unsubscribing: %s" % e)
            
        self.client.disconnect()
Christophe Cleva's avatar
Christophe Cleva committed
122
        #----- PROTECTED REGION END -----#	//	MetadataManager.delete_device
123
    
Christophe Cleva's avatar
Christophe Cleva committed
124 125 126

    def init_device(self):
        self.get_device_properties(self.get_device_class())
127 128
        self.attr_proposal_read = ''
        self.attr_sampleName_read = ''
Matias Guijarro's avatar
Matias Guijarro committed
129
        self.attr_datasetName_read = ''
130
        self.attr_metadataFile_read = ''
131
        self.attr_lastDatafile_read = ''
132 133
        self.attr_dataFolder_read = ''
        self.attr_sampleParameters_read = ['']
134
        self.attr_datafileList_read = ['']
Matias Guijarro's avatar
Matias Guijarro committed
135
        self.attr_datasetNameList_read = ['']
Christophe Cleva's avatar
Christophe Cleva committed
136
        #----- PROTECTED REGION ID(MetadataManager.init_device) ENABLED START -----#
137
        # init members
138 139 140 141 142 143 144 145 146 147 148
        self.client = None
        self.proxy = None
        self.running = False
        self.startDate = None
        self.endDate = None
        self.metadataFileName = None
        self.expdev = None
        self.proposalEventId = None
        self.sampleEventId = None
        self.dataRootEventId = None
        self.dataRoot = ''
149
        self.datasetParentLocation = ''
150 151 152 153 154 155 156 157 158
        # Priority of dataset metadata keys
        # (highest number is highest priority)
        # 1. parameters(local): from attributes of this device
        # 2. labels: from device property
        # 3. parameters(local): from attributes of other devices
        # 4. custom: added with SetParameters
        # 5. sample:
        # 6. globals: proposal, beamline, etc.
        self.datasetParamMap = dict()  # priority (local:1, remote:3)
159
        self.parameterDefinitionMap = dict()
160
        self.attributeProxyMap = dict()
161 162 163
        self.labels_map = dict()  # priority (2)
        self.customParamMap = dict() # priority (4)
        self.sampleParamMap = dict() # priority (5)
164
        self.attr_datafileList_read = []
Matias Guijarro's avatar
Matias Guijarro committed
165
        self.attr_datasetNameList_read = []
166
        self.msgList = deque(maxlen=20)
167
        
168
        # logger bridge
169 170
        logging.getLogger().setLevel(logging.DEBUG)
        self.logger = logging.getLogger(self.get_name())
171 172
        self.logger.addHandler(TangoLoggingHandler(self))
        if graypy != None:
173
            # this depends on the version of graypy used
174
            try:
175
                self.logger.addHandler(graypy.GELFHandler(self.graylogServer, self.graylogPort))                
176
            except:
177
                self.logger.addHandler(graypy.GELFUDPHandler(self.graylogServer, self.graylogPort))            
178
            
179
        # self client
180
        self.proxy = tango.DeviceProxy(self.get_name())
181
        # JMS client
182
        # we need to proxy to be created
183
        self.client = StompClient(self.queueURLs, self.queueName, self.beamlineID, manager=self, jolokia_port=self.jolokiaPort)
184
        self.logger.info("Init MetadataManager. url=%s, queue=%s metadatamanager=%s beamline=%s" % (self.client.getConfigURL(), self.queueName, self.get_name(), self.beamlineID))
185
        self.Connect()
186
        # MetaExp client        
187
        self.expdev = tango.DeviceProxy(self.metaExperimentDevice)
188 189 190
        self.proposalEventId = self.expdev.subscribe_event("Proposal", tango.EventType.CHANGE_EVENT, self, stateless=True)
        self.sampleEventId = self.expdev.subscribe_event("Sample", tango.EventType.CHANGE_EVENT, self, stateless=True)
        self.dataRootEventId = self.expdev.subscribe_event("DataRoot", tango.EventType.CHANGE_EVENT, self, stateless=True)
191
        # replicate attributes initialisation without calling write_XXX methods
192
        db = tango.Util.instance().get_database()
Matias Guijarro's avatar
Matias Guijarro committed
193 194
        attr_map = db.get_device_attribute_property(self.get_name(),['datasetName','dataFolder','metadataFile'])
        self.attr_datasetName_read = attr_map['datasetName'].get('__value',[''])[0]        
195 196
        self.attr_dataFolder_read = attr_map['dataFolder'].get('__value',[''])[0]
        self.attr_metadataFile_read = attr_map['metadataFile'].get('__value',[''])[0]
197
        if self.attr_proposal_read:
198
            self.info_stream("Recovered proposal %s" % self.attr_proposal_read)
199
        if self.attr_sampleName_read:
200
            self.info_stream("Recovered sample name %s" % self.attr_sampleName_read)
201
        if self.attr_datasetName_read:
202
            self.info_stream("Recovered dataset name %s" % self.attr_datasetName_read)
203
        if self.attr_dataFolder_read:
204
            self.info_stream("Recovered data folder %s" % self.attr_dataFolder_read)
205
        if self.attr_metadataFile_read:
206
            self.info_stream("Recovered metadata file %s" % self.attr_metadataFile_read)
207

208
        self.info_stream("Device init. Proposal=%s sample=%s dataset=%s dataFolder=%s metadataFile=%s" % (self.attr_proposal_read, self.attr_sampleName_read, self.attr_datasetName_read, self.attr_dataFolder_read, self.attr_metadataFile_read))
209 210
        # update state based on attributes
        self.UpdateState()
211 212 213 214 215 216 217 218 219 220 221
        
     
    '''
    Some getters for me to easily retrieve the info we will most likely use to log messages
    '''
    def get_proposal(self):
        return self.attr_proposal_read
    
    def get_sample_name(self):
        return self.attr_sampleName_read
         
Matias Guijarro's avatar
Matias Guijarro committed
222 223
    def get_dataset_name(self):
        return self.attr_datasetName_read
224 225 226
    
    def get_dataroot(self):
        return self.dataRoot           
Christophe Cleva's avatar
Christophe Cleva committed
227 228

    def always_executed_hook(self):
229
        return 
Christophe Cleva's avatar
Christophe Cleva committed
230 231 232
        #----- PROTECTED REGION ID(MetadataManager.always_executed_hook) ENABLED START -----#
        #----- PROTECTED REGION END -----#	//	MetadataManager.always_executed_hook

233
    #--------------------------------------------------------------------------
Christophe Cleva's avatar
Christophe Cleva committed
234
    #    MetadataManager read/write attribute methods
235
    #--------------------------------------------------------------------------
Christophe Cleva's avatar
Christophe Cleva committed
236
    
237
    def read_beamlineID(self, attr):
238
        attr.set_value(self.beamlineID.lower())  
239

240
    def read_proposal(self, attr):
241
        attr.set_value(self.attr_proposal_read)       
242
        
Matias Guijarro's avatar
Matias Guijarro committed
243
    def read_datasetState(self, attr):
244
        attr.set_value(str(self.dev_state()))   
245
        
246
    def is_proposal_allowed(self, attr):
247 248
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT])
249
        else:
250 251
            state_ok = not(self.get_state() in [tango.DevState.FAULT,
                tango.DevState.RUNNING])
252 253
        return state_ok
        
254
    def read_sampleName(self, attr):
255 256
        attr.set_value(self.attr_sampleName_read)  
        self.logger.info("read_sampleName. sampleName=%s metadatamanager=%s" % (self.attr_sampleName_read, self.get_name()))      
257
        
258
    def is_sampleName_allowed(self, attr):
259 260 261
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT,
                tango.DevState.OFF])
262
        else:
263 264 265
            state_ok = not(self.get_state() in [tango.DevState.FAULT,
                tango.DevState.OFF,
                tango.DevState.RUNNING])
266 267
        return state_ok
        
Matias Guijarro's avatar
Matias Guijarro committed
268 269
    def read_datasetName(self, attr):        
        attr.set_value(self.attr_datasetName_read)                
270
        
Matias Guijarro's avatar
Matias Guijarro committed
271 272 273 274 275
    def write_datasetName(self, attr):
        datasetName = attr.get_write_value()
        #if datasetName in self.attr_datasetNameList_read:
        #    self.logger.error("datasetName already used. datasetName=%s metadatamanager=%s" % (datasetName, self.get_name()))
        #    tango.Except.throw_exception("datasetName already used: %s" % datasetName, "datasetNames have to be unique for a proposal", "datasetName")
276
        if self.is_dataFolder_allowed(tango.AttReqType.WRITE_REQ):
277 278
            self.proxy.write_attribute("dataFolder", "")
        self.attr_dataFolder_read=""
279
        if self.is_metadataFile_allowed(tango.AttReqType.WRITE_REQ):
280 281
            self.proxy.write_attribute("metadataFile", "")
        self.attr_metadataFile_read=""
Matias Guijarro's avatar
Matias Guijarro committed
282
        self.attr_datasetName_read=datasetName
283
        self.UpdateState()
284
        
Matias Guijarro's avatar
Matias Guijarro committed
285
    def is_datasetName_allowed(self, attr):
286 287
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT, tango.DevState.OFF])
288
        else:
289
            state_ok = not(self.get_state() in [tango.DevState.FAULT, tango.DevState.OFF, tango.DevState.RUNNING])
290 291 292 293 294 295
        if state_ok and self.attr_sampleName_read:
            state_ok = True
        else:
            state_ok = False
        return state_ok
        
296
    def read_metadataFile(self, attr):
297
        if self.attr_metadataFile_read :
298
            attr.set_value(self.metadataWorker.Sanitize(self.attr_metadataFile_read))
299
        else :
Matias Guijarro's avatar
Matias Guijarro committed
300
            attr.set_value(self.metadataWorker.DefaultFileName(self.attr_proposal_read, self.attr_sampleName_read, self.attr_datasetName_read))
301 302
        
    def write_metadataFile(self, attr):
303
        self.attr_metadataFile_read= attr.get_write_value()
304
        
305
    def is_metadataFile_allowed(self, attr):
306 307
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT, tango.DevState.OFF, tango.DevState.STANDBY])
308
        else:
309
            state_ok = not(self.get_state() in [tango.DevState.FAULT, tango.DevState.OFF,tango.DevState.STANDBY,tango.DevState.RUNNING])
310 311
        return state_ok
        
312 313 314 315
    def read_lastDatafile(self, attr):
        attr.set_value(self.attr_lastDatafile_read)
        
    def write_lastDatafile(self, attr):
316
        data = attr.get_write_value()
317 318
        if data and data != self.attr_lastDatafile_read:
            self.attr_datafileList_read.append(data)
319 320
        self.attr_lastDatafile_read = data
        
321
    def is_lastDatafile_allowed(self, attr):
322 323
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT,tango.DevState.ON,tango.DevState.OFF,tango.DevState.STANDBY])
324
        else:
325
            state_ok = not(self.get_state() in [tango.DevState.FAULT,tango.DevState.ON,tango.DevState.OFF,tango.DevState.STANDBY])
326 327
        return state_ok
        
328
    def read_dataFolder(self, attr):
329
        if not self.attr_dataFolder_read:
330
            self.attr_dataFolder_read = self.DefaultFolder()
331 332
        attr.set_value(self.attr_dataFolder_read)
        
333
    def write_dataFolder(self, attr):
334
        self.attr_dataFolder_read = attr.get_write_value()
335
        
336
    def is_dataFolder_allowed(self, attr):
337 338
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT])
339
        else:
340
            state_ok = not(self.get_state() in [tango.DevState.FAULT,tango.DevState.OFF,tango.DevState.STANDBY,tango.DevState.RUNNING])
341 342
        return state_ok
        
bliss administrator's avatar
bliss administrator committed
343 344 345 346 347 348 349 350
    def read_CheckParameters(self):
        self.debug_stream("In read_CheckParameters()")

    def is_CheckParameters_allowed(self):
        self.debug_stream("In is_CheckParameters_allowed()")
        return True
      
    def CheckParameters(self):
351
        reader = ICATParameterReader(self.get_name(), self.authenticationPlugin, self.username, self.password, self.server, self.port)
bliss administrator's avatar
bliss administrator committed
352 353
        return str(reader.compare(self.parameters, self.labels))

354 355 356
    # returns the sample parameters set by the SetSampleParameters command as key value pairs
    # the list is dynamically created on each call        
    # returns "None" if list is empty
357 358 359 360 361 362 363 364
    def read_sampleParameters(self, attr):
        self.attr_sampleParameters_read=[]
        for k in self.sampleParamMap:
            self.attr_sampleParameters_read.append(k)
            self.attr_sampleParameters_read.append(self.sampleParamMap[k])
        if not self.attr_sampleParameters_read:
            self.attr_sampleParameters_read.append("None")
        attr.set_value(self.attr_sampleParameters_read)        
365
        
366
    def is_sampleParameters_allowed(self, attr):
367 368 369
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT,
                tango.DevState.OFF])
370 371 372 373 374 375 376 377
        else:
            state_ok = not(self.get_state() in [])
        if state_ok and self.attr_sampleName_read:
            state_ok = True
        else:
            state_ok = False
        return state_ok
        
378 379 380 381 382 383
    def read_datafileList(self, attr):
        if self.attr_datafileList_read:
            attr.set_value(self.attr_datafileList_read)
        else :
            attr.set_value(["None"])
        
384
    def is_datafileList_allowed(self, attr):
385 386 387 388 389
        if attr==tango.AttReqType.READ_REQ:
            state_ok = not(self.get_state() in [tango.DevState.FAULT,
                tango.DevState.ON,
                tango.DevState.OFF,
                tango.DevState.STANDBY])
390 391 392 393 394 395 396
        else:
            state_ok = not(self.get_state() in [])
        #----- PROTECTED REGION ID(MetadataManager.is_datafileList_allowed) ENABLED START -----#
        
        #----- PROTECTED REGION END -----#	//	MetadataManager.is_datafileList_allowed
        return state_ok
        
Matias Guijarro's avatar
Matias Guijarro committed
397 398 399 400
    def read_datasetNameList(self, attr):
        #----- PROTECTED REGION ID(MetadataManager.datasetNameList_read) ENABLED START -----#
        attr.set_value(self.attr_datasetNameList_read)
        #----- PROTECTED REGION END -----#	//	MetadataManager.datasetNameList_read
401
        
402 403 404 405 406
    def read_messageList(self, attr):
        #----- PROTECTED REGION ID(MetadataManager.messageList_read) ENABLED START -----#
        attr.set_value(self.msgList)
        #----- PROTECTED REGION END -----#    //    MetadataManager.messageList_read
        
Christophe Cleva's avatar
Christophe Cleva committed
407
    
408
    def read_parameter(self, attr):
409 410 411 412 413 414 415 416
        try:
            paramdef = self.parameterDefinitionMap.get(attr.get_name(), '')
            if paramdef.isRemote():
                attr.set_value(paramdef.getValue(self.attributeProxyMap))
            else:
                attr.set_value(self.datasetParamMap.get(attr.get_name(), ''))
        except:
            traceback.print_exc(file=sys.stdout)
417
        
418 419 420
    def read_label(self, attr):
        attr.set_value(self.labels_map[attr.get_name()])

421
    def write_parameter(self, attr):
422
        self.debug_stream("In write_parameter()")
423
        data = attr.get_write_value()
424 425
        self.datasetParamMap[attr.get_name()]=data
        
Christophe Cleva's avatar
Christophe Cleva committed
426
    
427 428 429
    def initialize_dynamic_attributes(self):
        # create dynamic parameters as defined in the device property
        # these need to be set_memorized_init(True) to be registered in the map on initialisation
430 431 432 433 434 435 436 437
        if self.parameters:
            for p in self.parameters:
                # create parameter definiton
                d = ParameterDefinition(p)
                pname = d.getParameterName()
                self.parameterDefinitionMap[pname] = d
                # add attributes: local -> RW, remote -> RO
                if d.isLocal():
438
                    self.info_stream("Local attribute: %s" %pname)
439
                    param = tango.Attr(pname, tango.DevString, tango.READ_WRITE)
440
                    param.set_memorized()
441 442
                    #param.set_memorized_init(True)
                    param.set_memorized_init(False)
443 444 445 446
                    self.add_attribute(param, MetadataManager.read_parameter, MetadataManager.write_parameter, None)
                    # register an empty value on creation, will be populated on initialization
                    self.datasetParamMap[pname] = ''
                else:
447
                    self.info_stream("Remote attribute: %s - from: %s" % (pname, d.getParameterDefinition()))
448
                    param = tango.Attr(pname, tango.DevString, tango.READ)
449 450 451 452 453
                    self.add_attribute(param, MetadataManager.read_parameter, None, None)
                    # create device proxies for all remote attributes
                    for dev in d.getRemoteAttributes():
                        if dev not in self.attributeProxyMap:
                            try:
454
                                self.attributeProxyMap[dev] = tango.AttributeProxy(dev)
455 456 457 458 459 460
                            except Exception as e:
                                self.error_stream("Error getting attribute proxy for %s: %s" %(dev, e))
        if self.labels:    
            for l in self.labels:
                aa = l.split("=")
                if len(aa) > 1:
461
                    pname = aa[0].strip()
462
                    self.info_stream("Label: %s" % pname)
463
                    param = tango.Attr(pname, tango.DevString, tango.READ)
464
                    self.add_attribute(param, MetadataManager.read_label, None, None)
465
                    self.labels_map[pname] = aa[1]
466
            
Christophe Cleva's avatar
Christophe Cleva committed
467 468
            
    def read_attr_hardware(self, data):
469
        self.debug_stream("In read_attr_hardware()")
Christophe Cleva's avatar
Christophe Cleva committed
470 471


472
    #--------------------------------------------------------------------------
Christophe Cleva's avatar
Christophe Cleva committed
473
    #    MetadataManager command methods
474
    #--------------------------------------------------------------------------
Christophe Cleva's avatar
Christophe Cleva committed
475
    
476 477 478
    def dev_state(self):
        """ This command gets the device state (stored in its device_state data member) and returns it to the caller.
        :return: Device state
479
        :rtype: tango.CmdArgType.DevState
480
        """
481
        self.debug_stream("In dev_state()")
482
        argout = tango.DevState.UNKNOWN
483 484 485
        #----- PROTECTED REGION ID(MetadataManager.State) ENABLED START -----#
        self.Connect()
        #----- PROTECTED REGION END -----#	//	MetadataManager.State
486
        if argout != tango.DevState.ALARM:
487
            tango.LatestDeviceImpl.dev_state(self)
488 489
        return self.get_state()
        
490 491 492 493 494
    def dev_status(self):
        """ This command gets the device status (stored in its device_status data member) and returns it to the caller.
        :return: Device status
        :rtype: str
        """
495
        self.debug_stream("In dev_status()")
496 497 498
        #----- PROTECTED REGION ID(MetadataManager.Status) ENABLED START -----#
        self.Connect()
        #----- PROTECTED REGION END -----#    //    MetadataManager.Status
499
        self.__status = tango.LatestDeviceImpl.dev_status(self)
500 501
        return self.__status

502
    def IngestTomodbXML(self, argin):
503 504
        """ Start the ingestion of the given tomodb XML file
        :param argin: Absolute path to the tomodb XML file
505
        :type argin: tango.DevString
506
        """
507
        self.debug_stream("In IngestTomodbXML()")
508
        #----- PROTECTED REGION ID(MetadataManager.IngestTomodbXML) ENABLED START -----#
509
        tomo = tomodbFile(self.attr_proposal_read, self.beamlineID.lower(), argin)
510
        self.client.sendObject(tomo)
511
        #----- PROTECTED REGION END -----#	//	MetadataManager.IngestTomodbXML
512

513
    def is_IngestTomodbXML_allowed(self):
514
        self.debug_stream("In is_IngestTomodbXML_allowed()")
515 516 517 518
        state_ok = not(self.get_state() in [tango.DevState.FAULT,
            tango.DevState.OFF,
            tango.DevState.STANDBY,
            tango.DevState.RUNNING])
519
        #----- PROTECTED REGION ID(MetadataManager.is_IngestTomodbXML_allowed) ENABLED START -----#
520
        
521
        #----- PROTECTED REGION END -----#	//	MetadataManager.is_IngestTomodbXML_allowed
522 523
        return state_ok
        
524 525
    def SetSampleParameters(self, argin):
        """ Sets the sample parameters, input is an array of successive key-value pairs
526
        :param argin: 
527
        :type argin: tango.DevVarStringArray
528
        """
529
        self.debug_stream("In SetSampleParameters()")
530
        #----- PROTECTED REGION ID(MetadataManager.SetSampleParameters) ENABLED START -----#
531
        if len(argin) % 2 != 0:
532
            self.warn_stream("Sample Parameters have to be key-value pairs, last value will be missing")
533 534
            argin.append("missing")
        for i in range(len(argin) / 2):
535 536
            self.sampleParamMap[argin[i*2]] = argin[i*2+1]
        #----- PROTECTED REGION END -----#	//	MetadataManager.SetSampleParameters
537
        
538
    def is_SetSampleParameters_allowed(self):
539
        self.debug_stream("In is_SetSampleParameters_allowed()")
540 541 542
        state_ok = not(self.get_state() in [tango.DevState.FAULT,
            tango.DevState.OFF,
            tango.DevState.RUNNING])
543 544 545 546 547 548 549
        #----- PROTECTED REGION ID(MetadataManager.is_SetSampleParameters_allowed) ENABLED START -----#
        if state_ok and self.attr_sampleName_read:
            state_ok = True
        else:
            state_ok = False
        #----- PROTECTED REGION END -----#	//	MetadataManager.is_SetSampleParameters_allowed
        return state_ok
550
    
551
    def GetDefaultFolder(self):
552 553
        return self.DefaultFolder()
    
554
    def GetDataFolderPattern(self):
555 556
        return self.dataFolderPattern
    
557
    def SetDatasetParentLocation(self, datasetParentLocation):  
558
        self.datasetParentLocation = datasetParentLocation
559
        self.info_stream("Set datasetParentLocation. datasetParentLocation=%s metadatamanager=%s" % (self.datasetParentLocation, self.get_name()))
560
        
561
    def GetDatasetParentLocation(self):        
562 563
        return self.datasetParentLocation  
        
564
    def ClearDatasetParentLocation(self):
565
        self.datasetParentLocation = ''
566
        self.info_stream("Clear datasetParentLocation. datasetParentLocation=%s metadatamanager=%s" % (self.datasetParentLocation, self.get_name()))
567
        
Matias Guijarro's avatar
Matias Guijarro committed
568 569
    def StartDataset(self):
        """ Indicates the beginning of a dataset. Sets the device in the RUNNING state.
570
        """
571 572 573 574 575
        try:        
            self.expdev.lock(tango.constants.DEFAULT_LOCK_VALIDITY)
            self.running = True
            self.set_state(tango.DevState.RUNNING)
            self.startDate = datetime.now()
576

577 578
            self.entries = dict()
            self.files = []
579
        
580 581 582
            self.logger.info("Dataset is starting. proposal=%s sample=%s datasetName=%s metadatamanager=%s datasetFolder=%s" % (self.get_proposal(), self.get_sample_name(), self.get_dataset_name(), self.get_name(), self.DefaultFolder()))        
            tangoworker.execute(self._export_initial)
            self.logger.info("Dataset is started. proposal=%s sample=%s datasetName=%s metadatamanager=%s datasetFolder=%s" % (self.get_proposal(), self.get_sample_name(), self.get_dataset_name(), self.get_name(), self.DefaultFolder()))
583
      
584 585
            for datasetParam in self.datasetParamMap:
                self.datasetParamMap[datasetParam] = ''
586
            self.customParamMap = dict()
587 588
        except:
            traceback.print_exc(file=sys.stdout)
589
        
Matias Guijarro's avatar
Matias Guijarro committed
590
    def is_StartDataset_allowed(self):
591
        return not(self.get_state() in [tango.DevState.FAULT,tango.DevState.OFF,tango.DevState.STANDBY,tango.DevState.RUNNING])
592
        
Matias Guijarro's avatar
Matias Guijarro committed
593 594
    def EndDataset(self):
        """ Indicates the end of the dataset in normal conditions. Triggers reporting of metadata.
595
        """
596
        self.endDate = datetime.now()
597
        dataset = self.CreateDataset()
598
        
Matias Guijarro's avatar
Matias Guijarro committed
599
        self.logger.info("Dataset is ending. proposal=%s sample=%s datasetName=%s metadatamanager=%s datasetFolder=%s" % (self.get_proposal(), self.get_sample_name(), self.get_dataset_name(), self.get_name(), self.DefaultFolder()))
600
        tangoworker.execute(self._export_final, dataset, self.metadataFileName, self.entries, self.files)
601 602
        # cleanup
        self.CleanupDatafiles()
603 604
        self.running = False
        self.UpdateState()
Matias Guijarro's avatar
Matias Guijarro committed
605 606
        # keep dataset name in list
        self.attr_datasetNameList_read.append(self.attr_datasetName_read)
607
        # clean the dataset
608
        self.logger.info("Dataset has ended. proposal=%s sample=%s datasetName=%s metadatamanager=%s datasetFolder=%s" % (self.get_proposal(), self.get_sample_name(), self.get_dataset_name(), self.get_name(), self.DefaultFolder()))
Matias Guijarro's avatar
Matias Guijarro committed
609
        self.proxy.write_attribute("datasetName", "")
610
        self.expdev.unlock()
611
        
Matias Guijarro's avatar
Matias Guijarro committed
612
    def is_EndDataset_allowed(self):
613
        self.debug_stream("In is_EndDataset_allowed()")
614 615 616 617
        state_ok = not(self.get_state() in [tango.DevState.FAULT,
            tango.DevState.ON,
            tango.DevState.OFF,
            tango.DevState.STANDBY])
618
        return state_ok
619

620
    # This sends a notification to the electronic logbook (ICAT+)
621
    def sendNotification(self, eventType, category, message):
622 623
         try:            
            if requests:
624 625 626
                # The content of the message
                content = [{"format" : "plainText", "text" : message}]
                datasetName = None
Matias Guijarro's avatar
Matias Guijarro committed
627 628
                if self.attr_datasetName_read :
                    datasetName = self.attr_datasetName_read
629
                
630 631
                params = dict(
                   {                        
632
                       "type"           : eventType,
633
                       "datasetName"    : datasetName,
634
                       "category"       : category,
635
                       "content"        : content,
636 637
                       "creationDate"   : str(datetime.now().isoformat())
                    }
638 639
                )          
                
640 641
                # Some versions of requests have not got this configuration
                #requests.adapters.DEFAULT_RETRIES = 1
642
                headers = {'Content-Type': 'application/json', 'Accept':'application/json'}   
643
                url = self.icatplus_server + "/logbook/" + str(self.API_KEY) + "/investigation/name/" + str(self.get_proposal()) + "/instrument/name/" + str(self.beamlineID.lower()) + "/event"
644 645 646
                self.logger.debug(url)
                requests.post(url, 
                data=(json.dumps(params)),
647 648
                headers=headers,
                timeout=0.1)
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
            else:
                self.logger.info("No requests installed")
         except Exception as e:
             sys.excepthook(*sys.exc_info())
             pass

    def uploadBase64(self, base64):                
        try:            
            if requests: 
                params = dict(
                   {                        
                       "base64"         : base64,
                       "creationDate"   : str(datetime.now().isoformat())
                    }
                )          
                
                headers = {'Content-Type': 'application/json', 'Accept':'application/json'}   
                url = self.icatplus_server + "/logbook/" + str(self.API_KEY) + "/investigation/name/" + str(self.get_proposal()) + "/instrument/name/" + str(self.beamlineID.lower()) + "/event/createFrombase64"
                self.logger.debug(url)
668
                requests.post(url, 
669
                data=(json.dumps(params)),
670 671
                headers=headers, 
                timeout=0.1)
672 673 674
            else:
                self.logger.info("No requests installed")
        except Exception as e:
675
             sys.excepthook(*sys.exc_info())
676
             pass
677 678

    def notifyInfo(self, message):                
679
        self.sendNotification("notification", "info", message)
680 681

    def notifyDebug(self, message):                
682
        self.sendNotification("notification", "debug", message)
683

684
    def notifyCommand(self, message):                
685
        self.sendNotification("notification", "commandLine", message)
686 687

    def notifyError(self, message):                
688 689 690 691
        self.sendNotification("notification", "error", message)

    def userComment(self, message):                
        self.sendNotification("annotation", "comment", message)
692

Matias Guijarro's avatar
Matias Guijarro committed
693
    def InterruptDataset(self):
694
        self.debug_stream("In InterruptDataset()")
695 696
        self.endDate = datetime.now()
        dts = self.CreateDataset(False)
697
        tangoworker.execute(self._export_final, dts, self.metadataFileName, self.entries, self.files)
698 699 700
        self.CleanupDatafiles()
        self.running = False
        self.UpdateState()
Matias Guijarro's avatar
Matias Guijarro committed
701 702
        self.attr_datasetNameList_read.append(self.attr_datasetName_read)
        self.proxy.write_attribute("datasetName", "")
703
        self.expdev.unlock()
704
        self.debug_stream("InterruptDataset() done")
Matias Guijarro's avatar
Matias Guijarro committed
705
        #----- PROTECTED REGION END -----#    //    MetadataManager.EndDataset
706
        
Matias Guijarro's avatar
Matias Guijarro committed
707
    def is_InterruptDataset_allowed(self):
708
        self.debug_stream("In is_EndDataset_allowed()")
709 710 711 712
        state_ok = not(self.get_state() in [tango.DevState.FAULT,
            tango.DevState.ON,
            tango.DevState.OFF,
            tango.DevState.STANDBY])
Matias Guijarro's avatar
Matias Guijarro committed
713
        #----- PROTECTED REGION ID(MetadataManager.is_EndDataset_allowed) ENABLED START -----#
714
        
Matias Guijarro's avatar
Matias Guijarro committed
715
        #----- PROTECTED REGION END -----#    //    MetadataManager.is_EndDataset_allowed
716 717
        return state_ok
    
Matias Guijarro's avatar
Matias Guijarro committed
718 719
    def AbortDataset(self):
        """ Indicates the abnormal termination of a running dataset. Does not report metadata.
720
        """
721
        self.debug_stream("In AbortDataset()")
Matias Guijarro's avatar
Matias Guijarro committed
722
        #----- PROTECTED REGION ID(MetadataManager.AbortDataset) ENABLED START -----#
723
        self.CleanupDatafiles()
724
        self.running = False
725
        self.set_state(tango.DevState.ON)
726
        self.expdev.unlock()
Matias Guijarro's avatar
Matias Guijarro committed
727
        #----- PROTECTED REGION END -----#	//	MetadataManager.AbortDataset
728
        
Matias Guijarro's avatar
Matias Guijarro committed
729
    def is_AbortDataset_allowed(self):
730
        self.debug_stream("In is_AbortDataset_allowed()")
731 732 733 734
        state_ok = not(self.get_state() in [tango.DevState.FAULT,
            tango.DevState.ON,
            tango.DevState.OFF,
            tango.DevState.STANDBY])
Matias Guijarro's avatar
Matias Guijarro committed
735
        #----- PROTECTED REGION ID(MetadataManager.is_AbortDataset_allowed) ENABLED START -----#
736
        
Matias Guijarro's avatar
Matias Guijarro committed
737
        #----- PROTECTED REGION END -----#	//	MetadataManager.is_AbortDataset_allowed
738 739 740 741
        return state_ok
        
    def Reset(self):
        """ Resets the device to the OFF state. Clears attributes.
742
        """
743
        self.debug_stream("In Reset()")
744
        #----- PROTECTED REGION ID(MetadataManager.Reset) ENABLED START -----#
Matias Guijarro's avatar
Matias Guijarro committed
745 746
        # clears the proposal, this cascade clears the sampleName, sample parameters and datasetName
        self.proxy.write_attribute("datasetName", "")
747 748 749
        #----- PROTECTED REGION END -----#	//	MetadataManager.Reset
        
    def is_Reset_allowed(self):
750
        self.debug_stream("In is_Reset_allowed()")
751 752
        state_ok = not(self.get_state() in [tango.DevState.FAULT,
            tango.DevState.RUNNING])
753 754 755 756 757
        #----- PROTECTED REGION ID(MetadataManager.is_Reset_allowed) ENABLED START -----#
        
        #----- PROTECTED REGION END -----#	//	MetadataManager.is_Reset_allowed
        return state_ok
        
758 759 760 761
    def addParametersToDataSet(self, dataset, parameters):
        for parameter in parameters:
             dataset.add_parameter(parameter)
        return dataset
762 763

    #----- PROTECTED REGION ID(MetadataManager.programmer_methods) ENABLED START -----#
764
    def _export_final(self, dataset, metadatafilename, entries, files, ):
Matias Guijarro's avatar
Matias Guijarro committed
765
        self.metadataWorker.exportFinal(dataset, metadatafilename, entries, files, self.attr_metadataFile_read, self.attr_proposal_read, self.attr_sampleName_read, self.attr_datasetName_read,  self.getParameters(), self.client)
766 767
       
    def _export_initial(self):
768
        try:
769
            
770 771 772 773 774 775
            # now we need to make sure the datafolder attribute is correct
            if not self.attr_dataFolder_read:
                self.attr_dataFolder_read = self.DefaultFolder()

            # record h5 file name for later stage
            if os.path.exists(self.attr_dataFolder_read) and os.access(self.attr_dataFolder_read, os.W_OK | os.X_OK):
Matias Guijarro's avatar
Matias Guijarro committed
776
                 self.metadataFileName = self.metadataWorker.MakeFileName(self.attr_dataFolder_read, "h5", self.attr_metadataFile_read, self.attr_proposal_read, self.attr_sampleName_read, self.attr_datasetName_read, True)
777
            else:
778
                self.warn_stream("Folder %s does not exist, or is not writable, will save data in local folder" % self.attr_dataFolder_read)
Matias Guijarro's avatar
Matias Guijarro committed
779
                self.metadataFileName = self.metadataWorker.MakeFileName("data", "h5",  self.attr_metadataFile_read, self.attr_proposal_read, self.attr_sampleName_read, self.attr_datasetName_read, False)
780
    
781 782 783 784 785 786 787

            # write the initial parameters
            self.endDate = None #datetime.now()
            dataset = self.CreateDataset()
            # we need to prepare everything for async processing
            self.entries = dict()
            self.files = []
788
          
789
            self.debug_stream("globalHDFfiles %s" % (self.globalHDFfiles))
790 791 792
            for f in self.globalHDFfiles:
                self.files.append(self._replaceInPath(f))
                entries = self.metadataWorker.exportInitial(dataset, self.metadataFileName, self.entries, self.files, self.getParameters(), self.sampleParamMap)
793
        except Exception as e:
794
            traceback.print_exc(file=sys.stdout)
795
            self.error_stream("Error exporting %s: %s" % (self.metadataFileName, e))            
796
          
797
            
798