Skip to content
Snippets Groups Projects
Commit 10105ac9 authored by Holger Witsch's avatar Holger Witsch
Browse files

proposalName back to proposal 

The changes with the purpose of adding session had the proposal attribute
renamed to proposalName, which, of course, breaks all the macros and
has no added value. The same with the sessions, the new attribute was
sessionName. Being new, that would be alright, but shorter seems better.
We already know it is a name.
parent 69858a22
No related branches found
No related tags found
1 merge request!41Resolve "Add the proposal session name"
Expand all Hide whitespace changes
Inline Side-by-side
src/metadata_manager/MetaExperiment.py
+ 70
− 70
View file @ 10105ac9
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# -*- coding:utf-8 -*-
##############################################################################
......@@ -7,24 +7,24 @@
##============================================================================
##
## File : MetaExperiment.py
##
##
## Project : Metadata for high level experiment
##
## This file is part of Tango device class.
##
##
## Tango is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
##
## Tango is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
##
## You should have received a copy of the GNU General Public License
## along with Tango. If not, see <http://www.gnu.org/licenses/>.
##
##
##
## $Author : andy.gotz$
##
......@@ -64,19 +64,19 @@ class MetaExperiment (tango.LatestDeviceImpl):
tango.LatestDeviceImpl.__init__(self,cl,name)
self.debug_stream("In __init__()")
MetaExperiment.init_device(self)
def delete_device(self):
self.debug_stream("In delete_device()")
self.debug_stream("In delete_device()")
self.client.disconnect()
def init_device(self):
self.debug_stream("In init_device()")
self.get_device_properties(self.get_device_class())
self.attr_proposalName_read = ''
self.attr_proposal_read = ''
self.attr_proposalSession_read = ''
self.attr_sample_read = ''
self.attr_dataRoot_read = ''
self.attr_proposalName_read = 'please enter'
self.attr_dataRoot_read = ''
self.attr_proposal_read = 'please enter'
self.attr_proposalSession_read = 'please enter'
self.attr_sample_read = 'please enter'
self.attr_dataRoot_read = '/data/visitor/'
......@@ -99,7 +99,7 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.logger.addHandler(graypy.GELFUDPHandler(self.graylogServer, self.graylogPort))
except:
traceback.print_exc(file=sys.stdout)
# JMS client
self.client = StompClient(self.queueURLs, self.queueName, self.beamlineID, jolokia_port=self.jolokiaPort)
self.info_stream("Configuration URL=%s" %self.client.getConfigURL())
......@@ -109,66 +109,66 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.proxy = tango.DeviceProxy(self.get_name())
# mimic write_hardware without side effects
db = tango.Util.instance().get_database()
attr_map = db.get_device_attribute_property(self.get_name(),['proposalName','proposalSession','sample','dataRoot'])
self.attr_proposalName_read = attr_map['proposalName'].get('__value',[''])[0]
self.attr_proposalSession_read = attr_map['proposalSession'].get('__value',[''])[0]
attr_map = db.get_device_attribute_property(self.get_name(),['proposal','proposalSession','sample','dataRoot'])
self.attr_proposal_read = attr_map['proposal'].get('__value',[''])[0]
self.attr_proposalSession_read = attr_map['proposalSession'].get('__value',[''])[0]
self.attr_sample_read = attr_map['sample'].get('__value',[''])[0]
self.attr_dataRoot_read = attr_map['dataRoot'].get('__value',[''])[0]
if self.attr_proposalName_read:
self.info_stream("Recovered proposal name %s" %self.attr_proposalName_read)
if self.attr_proposalSession_read:
if self.attr_proposal_read:
self.info_stream("Recovered proposal name %s" %self.attr_proposal_read)
if self.attr_proposalSession_read:
self.info_stream("Recovered proposal session %s" %self.attr_proposalSession_read)
if self.attr_sample_read:
self.info_stream("Recovered sample name %s" %self.attr_sample_read)
if self.attr_dataRoot_read:
self.info_stream("Recovered data root %s" %self.attr_dataRoot_read)
# we need to push them in this order
self.push_change_event("ProposalName", self.attr_proposalName_read)
self.push_change_event("ProposalName", self.attr_proposal_read)
self.push_change_event("ProposalSession", self.attr_proposalSession_read)
self.push_change_event("Sample", self.attr_sample_read)
self.push_change_event("DataRoot", self.attr_dataRoot_read)
self.UpdateState()
#----- PROTECTED REGION END -----# // MetaExperiment.init_device
#----- PROTECTED REGION END -----# // MetaExperiment.init_device
def always_executed_hook(self):
self.debug_stream("In always_executed_hook()")
#----- PROTECTED REGION ID(MetaExperiment.always_executed_hook) ENABLED START -----#
#----- PROTECTED REGION END -----# // MetaExperiment.always_executed_hook
#----- PROTECTED REGION END -----# // MetaExperiment.always_executed_hook
#--------------------------------------------------------------------------
# MetaExperiment read/write attribute methods
#--------------------------------------------------------------------------
def read_proposalName(self, attr):
self.debug_stream("In read_proposalName()")
def read_proposal(self, attr):
self.debug_stream("In read_proposal()")
#----- PROTECTED REGION ID(MetaExperiment.proposal_read) ENABLED START -----#
attr.set_value(self.attr_proposalName_read)
#----- PROTECTED REGION END -----# // MetaExperiment.proposal_read
def write_proposalName(self, attr):
self.debug_stream("In write_proposalName()")
attr.set_value(self.attr_proposal_read)
#----- PROTECTED REGION END -----# // MetaExperiment.proposal_read
def write_proposal(self, attr):
self.debug_stream("In write_proposal()")
data = attr.get_write_value()
#----- PROTECTED REGION ID(MetaExperiment.proposal_write) ENABLED START -----#
# new proposal (investigation) registration, if not empty
if data and (data != self.attr_proposalName_read):
if data and (data != self.attr_proposal_read):
self.CreateInvestigation(data)
self.attr_proposalName_read = data
self.push_change_event("ProposalName", self.attr_proposalName_read)
self.attr_proposal_read = data
self.push_change_event("ProposalName", self.attr_proposal_read)
self.proxy.write_attribute("sample", "please enter")
self.proxy.write_attribute("dataRoot", "/data/visitor/")
self.UpdateState()
#----- PROTECTED REGION END -----# // MetaExperiment.proposal_write
#----- PROTECTED REGION END -----# // MetaExperiment.proposal_write
def read_proposalSession(self, attr):
self.debug_stream("In read_proposalSession()")
#----- PROTECTED REGION ID(MetaExperiment.read_proposalSession) ENABLED START -----#
attr.set_value(self.attr_proposalSession_read)
#----- PROTECTED REGION END -----# // MetaExperiment.session_read
#----- PROTECTED REGION END -----# // MetaExperiment.session_read
def write_proposalSession(self, attr):
self.debug_stream("In write_proposalSession()")
data = attr.get_write_value()
......@@ -178,15 +178,15 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.proxy.write_attribute("sample", "please enter")
self.proxy.write_attribute("dataRoot", "/data/visitor/")
self.UpdateState()
#----- PROTECTED REGION END -----# // MetaExperiment.session_write
#----- PROTECTED REGION END -----# // MetaExperiment.session_write
def read_sample(self, attr):
self.debug_stream("In read_sample()")
#----- PROTECTED REGION ID(MetaExperiment.sample_read) ENABLED START -----#
attr.set_value(self.attr_sample_read)
#----- PROTECTED REGION END -----# // MetaExperiment.sample_read
#----- PROTECTED REGION END -----# // MetaExperiment.sample_read
def write_sample(self, attr):
self.debug_stream("In write_sample()")
data = attr.get_write_value()
......@@ -194,14 +194,14 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.attr_sample_read = data
self.push_change_event("Sample", self.attr_sample_read)
self.UpdateState()
#----- PROTECTED REGION END -----# // MetaExperiment.sample_write
#----- PROTECTED REGION END -----# // MetaExperiment.sample_write
def read_dataRoot(self, attr):
self.debug_stream("In read_dataRoot()")
#----- PROTECTED REGION ID(MetaExperiment.dataRoot_read) ENABLED START -----#
attr.set_value(self.attr_dataRoot_read)
#----- PROTECTED REGION END -----# // MetaExperiment.dataRoot_read
#----- PROTECTED REGION END -----# // MetaExperiment.dataRoot_read
def write_dataRoot(self, attr):
self.debug_stream("In write_dataRoot()")
data = attr.get_write_value()
......@@ -209,22 +209,22 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.attr_dataRoot_read = data
self.push_change_event("DataRoot", self.attr_dataRoot_read)
self.UpdateState()
#----- PROTECTED REGION END -----# // MetaExperiment.dataRoot_write
#----- PROTECTED REGION END -----# // MetaExperiment.dataRoot_write
def read_attr_hardware(self, data):
self.debug_stream("In read_attr_hardware()")
#----- PROTECTED REGION ID(MetaExperiment.read_attr_hardware) ENABLED START -----#
#----- PROTECTED REGION END -----# // MetaExperiment.read_attr_hardware
#----- PROTECTED REGION END -----# // MetaExperiment.read_attr_hardware
#--------------------------------------------------------------------------
# MetaExperiment command methods
#--------------------------------------------------------------------------
def dev_state(self):
""" This command gets the device state (stored in its device_state data member) and returns it to the caller.
:return: Device state
......@@ -234,11 +234,11 @@ class MetaExperiment (tango.LatestDeviceImpl):
argout = tango.DevState.UNKNOWN
#----- PROTECTED REGION ID(MetaExperiment.State) ENABLED START -----#
self.Connect()
#----- PROTECTED REGION END -----# // MetaExperiment.State
#----- PROTECTED REGION END -----# // MetaExperiment.State
if argout != tango.DevState.ALARM:
tango.LatestDeviceImpl.dev_state(self)
return self.get_state()
def dev_status(self):
""" This command gets the device status (stored in its device_status data member) and returns it to the caller.
:return: Device status
......@@ -271,44 +271,44 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.client.sendObject(inv)
def UpdateState(self):
if not self.attr_proposalName_read or (self.attr_proposalName_read == "please enter"):
if not self.attr_proposal_read or (self.attr_proposal_read == "please enter"):
self.set_status("No experiment started")
elif not self.attr_proposalSession_read or (self.attr_proposalSession_read == "please enter"):
self.set_status("No experiment started")
elif self.attr_sample_read and (self.attr_sample_read != "please enter"):
self.set_status("Ready to start dataset for experiment %r and session %r" % (self.attr_proposalName_read, self.attr_proposalSession_read))
self.set_status("Ready to start dataset for experiment %r and session %r" % (self.attr_proposal_read, self.attr_proposalSession_read))
else:
self.set_status("Experiment %r session %r started" % (self.attr_proposalName_read, self.attr_proposalSession_read))
self.set_status("Experiment %r session %r started" % (self.attr_proposal_read, self.attr_proposalSession_read))
try:
self.append_status(self.client.getStatus(),True)
except Exception:
self.debug_stream("Error appending status")
#----- PROTECTED REGION END -----# // MetaExperiment.programmer_methods
#----- PROTECTED REGION END -----# // MetaExperiment.programmer_methods
class MetaExperimentClass(tango.DeviceClass):
#--------- Add you global class variables here --------------------------
#----- PROTECTED REGION ID(MetaExperiment.global_class_variables) ENABLED START -----#
#----- PROTECTED REGION END -----# // MetaExperiment.global_class_variables
#----- PROTECTED REGION END -----# // MetaExperiment.global_class_variables
# Class Properties
class_property_list = {
'graylogServer':
[tango.DevString,
[tango.DevString,
"Graylog Server",
["graylog-dau.esrf.fr"] ],
'graylogPort':
[tango.DevInt,
[tango.DevInt,
"Graylog port",
[12203] ],
'queueURLs':
[tango.DevVarStringArray,
[tango.DevVarStringArray,
"URLs of the queues (including port)",
["bcu-mq-01.esrf.fr:61613"]],
'queueName':
[tango.DevString,
[tango.DevString,
"Name of the stomp queue for sending ingestion requests",
["/queue/icatIngest"] ],
'jolokiaPort':
......@@ -321,7 +321,7 @@ class MetaExperimentClass(tango.DeviceClass):
# Device Properties
device_property_list = {
'beamlineID':
[tango.DevString,
[tango.DevString,
"Beamline ID for reporting metadata",
[] ],
}
......@@ -334,7 +334,7 @@ class MetaExperimentClass(tango.DeviceClass):
# Attribute definitions
attr_list = {
'proposalName':
'proposal':
[[tango.DevString,
tango.SCALAR,
tango.READ_WRITE],
......@@ -371,8 +371,8 @@ def main():
py = tango.Util(sys.argv)
py.add_class(MetaExperimentClass, MetaExperiment, 'MetaExperiment')
#----- PROTECTED REGION ID(MetaExperiment.add_classes) ENABLED START -----#
#----- PROTECTED REGION END -----# // MetaExperiment.add_classes
#----- PROTECTED REGION END -----# // MetaExperiment.add_classes
U = tango.Util.instance()
U.server_init()
......
src/metadata_manager/MetadataManager.py
+ 205
− 205
View file @ 10105ac9
This diff is collapsed.
src/metadata_manager/MetadataWorker.py
+ 36
− 36
View file @ 10105ac9
......@@ -44,14 +44,14 @@ class MetadataWorker():
def __init__(self, metadataManagerName):
'''
Constructor
'''
'''
self.logger = logging.getLogger("MetadataWorker({})".format(metadataManagerName))
self.entries = []
self.entries = []
self.writer = HDF5FileWriter(metadataManagerName)
self.fileExplorer = FileExplorer()
self.fileExplorer = FileExplorer()
self.metadataManagerName = metadataManagerName
'''
'''
Parameters
'''
self.__technique = None
......@@ -63,14 +63,14 @@ class MetadataWorker():
def getTechnique(self):
return self.__technique
def createDataset(self, complete, attr_proposalName_read,beamlineID, attr_datasetName_read, attr_dataFolder_read, startDate,endDate, labels_map, datasetParamMap, attr_sampleName_read, sampleParamMap, attr_datafileList_read):
def createDataset(self, complete, attr_proposal_read,beamlineID, attr_datasetName_read, attr_dataFolder_read, startDate,endDate, labels_map, datasetParamMap, attr_sampleName_read, sampleParamMap, attr_datafileList_read):
self.attr_dataFolder_read = attr_dataFolder_read
# dataset basic parameters
dts = dataset(complete, attr_proposalName_read,
# dataset basic parameters
dts = dataset(complete, attr_proposal_read,
beamlineID,
attr_datasetName_read,
attr_dataFolder_read,
startDate, endDate)
startDate, endDate)
# add the dataset parameters
for n, v in datasetParamMap.items():
# register only non empty parameters
......@@ -82,8 +82,8 @@ class MetadataWorker():
if v:
dts.add_parameter(parameter(n, v))
# sample and sample parameters, if any
if attr_sampleName_read:
if attr_sampleName_read:
smpl = sample(attr_sampleName_read)
for n, v in sampleParamMap.items():
# register only non empty parameters
......@@ -94,7 +94,7 @@ class MetadataWorker():
dts.set_sample(smpl)
# datafiles, if any
# datafile path is created from data folder and file name
for f in attr_datafileList_read:
for f in attr_datafileList_read:
dts.add_datafile(datafile(os.path.join(attr_dataFolder_read, f)))
# return the created dataset object
return dts
......@@ -122,16 +122,16 @@ class MetadataWorker():
for parameter in parameters:
dataset.add_parameter(parameter)
return dataset
def exportInitial(self, dataset, metadatafilename, entries, files, parameters, sampleParamMap):
try:
self.entries = entries
try:
self.entries = entries
dataset = self.addParametersToDataSet(dataset, parameters)
# transform
params = self.toDictionary(dataset, sampleParamMap, initial=True)
# export single dataset file
self.logger.debug(self.entries)
# export single dataset file
self.logger.debug(self.entries)
self.entries[metadatafilename] = self.writer.appendTo(metadatafilename, params, HDF5FileWriter.PHASE_INITIAL, [])
self.logger.debug(self.entries)
......@@ -141,10 +141,10 @@ class MetadataWorker():
if HDF5FileWriter.ENTRY_KEY in params:
del params[HDF5FileWriter.ENTRY_KEY]
try:
self.logger.warn("Exporting parameters to master. file=%s metadatamanager=%s" %(f, self.metadataManagerName))
self.logger.warn("Exporting parameters to master. file=%s metadatamanager=%s" %(f, self.metadataManagerName))
entries[f] = self.writer.appendTo(f, params, HDF5FileWriter.PHASE_INITIAL, [])
except Exception as e:
self.logger.warn("Error exporting %s: %s. metadatamanager=%s" %(f, e, self.metadataManagerName))
self.logger.warn("Error exporting %s: %s. metadatamanager=%s" %(f, e, self.metadataManagerName))
except Exception as e:
traceback.print_exc(file=sys.stdout)
return self.entries
......@@ -153,13 +153,13 @@ class MetadataWorker():
def exportFinal(self, dataset, metadatafilename, entries, files, attr_metadataFile_read, attr_proposalName_read, attr_sampleName_read, attr_datasetName_read, parameters, client):
try:
# No more export to XML file
#xmlfilename = self.MakeFileName('data', 'xml', attr_metadataFile_read, attr_proposalName_read, attr_sampleName_read, attr_datasetName_read,False)
def exportFinal(self, dataset, metadatafilename, entries, files, attr_metadataFile_read, attr_proposal_read, attr_sampleName_read, attr_datasetName_read, parameters, client):
try:
# No more export to XML file
#xmlfilename = self.MakeFileName('data', 'xml', attr_metadataFile_read, attr_proposal_read, attr_sampleName_read, attr_datasetName_read,False)
dataset = self.addParametersToDataSet(dataset, parameters)
# send to ingestion
# send to ingestion
client.sendObject(dataset)
# keep local xml for reference
......@@ -167,8 +167,8 @@ class MetadataWorker():
# transform
sampleParamMap = []
params = self.toDictionary(dataset,sampleParamMap, initial=True)
# export single dataset file
# export single dataset file
if entries[metadatafilename] is not None:
self.logger.info("Exporting HDF5. metadatafilename=%s metadatamanager=%s" %(str(entries[metadatafilename]), self.metadataManagerName))
else:
......@@ -192,18 +192,18 @@ class MetadataWorker():
self.logger.warn("Error exporting. file=%s error=%s metadatamanager=%s" %(f, e, self.metadataManagerName))
else:
self.logger.warn("Not trying to export to %s, entry not registered. metadatamanager=%s" % (f, self.metadataManagerName))
except Exception as e:
traceback.print_exc(file=sys.stdout)
def DefaultFileName(self, attr_proposalName_read, attr_sampleName_read, attr_datasetName_read):
return self.Sanitize(attr_proposalName_read + "-" + attr_sampleName_read + "-" + attr_datasetName_read)
def DefaultFileName(self, attr_proposal_read, attr_sampleName_read, attr_datasetName_read):
return self.Sanitize(attr_proposal_read + "-" + attr_sampleName_read + "-" + attr_datasetName_read)
def Sanitize(self, filename):
return ''.join(c if c in self.validFilenameChars else '_' for c in filename)
def MakeFileName(self, folder, extension, attr_metadataFile_read, attr_proposalName_read, attr_sampleName_read, attr_datasetName_read, useGivenFilename=True):
def MakeFileName(self, folder, extension, attr_metadataFile_read, attr_proposal_read, attr_sampleName_read, attr_datasetName_read, useGivenFilename=True):
# prepend . if not present
ext = extension
if ext[0] != '.':
......@@ -214,16 +214,16 @@ class MetadataWorker():
# append extension if not present
if not fname.endswith(ext):
fname += ext
# otherwise use dataset name
# otherwise use dataset name
else:
fname = os.path.join(folder, self.DefaultFileName(attr_proposalName_read, attr_sampleName_read, attr_datasetName_read) + ext)
fname = os.path.join(folder, self.DefaultFileName(attr_proposal_read, attr_sampleName_read, attr_datasetName_read) + ext)
# check if file exists, append number if it does
x = 0
xname = fname
while os.path.isfile(xname):
self.logger.warn("File already exists, will use an incremented file name. file=%s metadatamanager=%s" % (xname, self.metadataManagerName))
xname = fname[:-(len(ext))] + '-' + str(x) + ext
x += 1
x += 1
return xname
# Returns the file on tehe output folder
......@@ -242,8 +242,8 @@ class MetadataWorker():
try:
f = open(filepath, 'w-')
# write the file
dataset.export(f, 0, pretty_print=True)
self.logger.info("Dataset exported as XML. filepath=%s metadatamanager=%s" % (filepath, self.metadataManagerName))
dataset.export(f, 0, pretty_print=True)
self.logger.info("Dataset exported as XML. filepath=%s metadatamanager=%s" % (filepath, self.metadataManagerName))
except Exception as e:
self.logger.error("Error writing metadata. file=%s error=%s metadatamanager=%s" % (filepath, e, self.metadataManagerName))
finally:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment