Commit 58a1c21b authored by Wout De Nolf's avatar Wout De Nolf

[logging] avoid using the root logger

parent ef2cb6fa
......@@ -31,8 +31,6 @@ class HDF5FileWriter():
ENTRY_KEY = 'entry'
_ENTRY_FORMAT = 'entry_{0:0>4d}: {1} - {2}'
LOG = logging.getLogger("HDF5FileWriter")
def __init__(self, metadataManagerName):
'''
Constructor
......@@ -40,6 +38,7 @@ class HDF5FileWriter():
self.root = ET.parse(HDF5FileWriter._DEFINITION_FILE_LOCATION).getroot()
self.snapshotName = None
self.metadataManagerName = metadataManagerName
self.logger = logging.getLogger("HDF5FileWriter({})".format(metadataManagerName))
'''
......@@ -70,7 +69,7 @@ class HDF5FileWriter():
if phase == HDF5FileWriter.PHASE_INITIAL:
entry = HDF5FileWriter._ENTRY_FORMAT.format(len(f.keys()), params.get('sampleName', 'NO_SAMPLE'), params.get('datasetName', 'NO_SCAN'))
#if HDF5FileWriter.ENTRY_KEY in params:
# self.LOG.warn("Replacing existing ENTRY %s by new one %s" % (params.get(HDF5FileWriter.ENTRY_KEY), entry))
# self.logger.warn("Replacing existing ENTRY %s by new one %s" % (params.get(HDF5FileWriter.ENTRY_KEY), entry))
params[HDF5FileWriter.ENTRY_KEY] = entry
elif phase == HDF5FileWriter.PHASE_FINAL:
if HDF5FileWriter.ENTRY_KEY not in params:
......@@ -81,24 +80,24 @@ class HDF5FileWriter():
self._writeNode(f, self.root, "", params, phase)
self._writeNode(f, self.root, "", params, HDF5FileWriter._LINK_PHASE_)
self._writeFiles(f, entry, outputFileList)
#self.LOG.info("Dataset %s exported to %s, %s phase" % (params.get('datasetName', ''), fileName, phase))
#self.logger.info("Dataset %s exported to %s, %s phase" % (params.get('datasetName', ''), fileName, phase))
return entry
except Exception as e:
#traceback.print_exc()
self.LOG.error("Error %s. metadatamanager=%s" % (e, self.metadataManagerName))
self.logger.error("Error %s. metadatamanager=%s" % (e, self.metadataManagerName))
finally:
if f:
try:
f.close()
except Exception as e:
self.LOG.warn("Error closing file %s: %s. metadatamanager=%s" % (fileName, e, self.metadataManagerName))
self.logger.warn("Error closing file %s: %s. metadatamanager=%s" % (fileName, e, self.metadataManagerName))
def _writeGroupNode(self, hdffile, currentNode, parentPath, params, phase):
name = currentNode.get(HDF5FileWriter._GROUP_NAME_ATTRIBUTE)
value = self._getValueFor(name, params)
currentPath = parentPath + "/" + value
#self.LOG.debug("Processing group %s (%s)" % (name, value))
#self.logger.debug("Processing group %s (%s)" % (name, value))
# Recursion on group children
for child in currentNode:
......@@ -108,16 +107,16 @@ class HDF5FileWriter():
g = hdffile.get(currentPath, None)
if g is not None:
if len(g.attrs) == 0:
#self.LOG.debug("Recording group %s (%s) attributes. metadatamanager=%s" % (name, value, self.metadataManagerName))
#self.logger.debug("Recording group %s (%s) attributes. metadatamanager=%s" % (name, value, self.metadataManagerName))
d = currentNode.attrib.copy()
d.pop(HDF5FileWriter._GROUP_NAME_ATTRIBUTE)
for k, v in d.items():
if not k.startswith(self._ESRF_ATTRIBUTE_HEADER):
g.attrs[k] = v
#else:
# self.LOG.debug("Group %s (%s) already created, skipping" % (name, value))
# self.logger.debug("Group %s (%s) already created, skipping" % (name, value))
#else:
# self.LOG.debug("Skipping empty group %s (%s)" % (name, value))
# self.logger.debug("Skipping empty group %s (%s)" % (name, value))
def _writeLinkNode(self, hdffile, currentNode, parentPath, params, phase):
if phase != HDF5FileWriter._LINK_PHASE_:
......@@ -152,7 +151,7 @@ class HDF5FileWriter():
def _writeParameterNode(self, hdffile, currentNode, parentPath, params, phase):
nodePhase = currentNode.get(HDF5FileWriter._RECORD_TYPE_ATTRIBUTE, HDF5FileWriter.PHASE_INITIAL)
# self.LOG.info("\t\t WRITE NODE: [{0}] [{1}] [{2}] [{3}] [{4}]]".format(str(currentNode.text), self._getValueFor(currentNode.text, params), str(parentPath), str(phase), str(nodePhase)))
# self.logger.info("\t\t WRITE NODE: [{0}] [{1}] [{2}] [{3}] [{4}]]".format(str(currentNode.text), self._getValueFor(currentNode.text, params), str(parentPath), str(phase), str(nodePhase)))
# Skip everything if not in phase
if phase != nodePhase:
return;
......@@ -173,24 +172,24 @@ class HDF5FileWriter():
value = self._getValueFor(currentNode.text, params)
# early return for missing param
if value is None:
#self.LOG.debug("Skipping field %s, no value for %s" % (name, currentNode.text))
#self.logger.debug("Skipping field %s, no value for %s" % (name, currentNode.text))
return;
# create field with correct type and value
#self.LOG.debug("Recording field %s = %s (%s)" % (name, currentNode.text, value))
#self.logger.debug("Recording field %s = %s (%s)" % (name, currentNode.text, value))
if dt is None or dt == 'NX_CHAR':
hdffile[currentPath] = numpy.string_(value)
elif dt == 'NX_DATE_TIME':
try:
hdffile[currentPath] = numpy.string_(datetime.isoformat(value))
except Exception as e:
self.LOG.error("Unable to format date value %s: %s" % (value, e))
self.logger.error("Unable to format date value %s: %s" % (value, e))
hdffile[currentPath] = numpy.string_("ERROR")
elif dt == 'NX_INT':
if "ERROR" != value and "None" != value:
try:
hdffile[currentPath] = int(value)
except Exception as e:
self.LOG.error("Unable to format int value %s: %s" % (value, e))
self.logger.error("Unable to format int value %s: %s" % (value, e))
hdffile[currentPath] = int("-9999")
else:
hdffile[currentPath] = int("-9999")
......@@ -199,7 +198,7 @@ class HDF5FileWriter():
try:
hdffile[currentPath] = float(value)
except Exception as e:
self.LOG.error("Unable to format double value %s: %s. metadatamanager=%s" % (value, e, self.metadataManagerName))
self.logger.error("Unable to format double value %s: %s. metadatamanager=%s" % (value, e, self.metadataManagerName))
hdffile[currentPath] = float("-9999")
else:
hdffile[currentPath] = float("-9999")
......
......@@ -7,7 +7,6 @@ import logging
from metadata_manager.ParameterDefinition import ParameterDefinition
class ICATParameterReader:
LOG = logging.getLogger("ICATParameterReader")
def __init__(self, metadatamanager, authenticationPlugin, username, password, server, port):
self.metadatamanager = metadatamanager
self.data = []
......@@ -16,6 +15,7 @@ class ICATParameterReader:
self.password = password
self.server = server
self.port = port
self.logger = logging.getLogger("ICATParameterReader({})".format(metadatamanager))
def __getSessionId(self, authPlugin, username, password, server, port ):
params = urllib.parse.urlencode({'json': '{"plugin":"' + authPlugin +'","credentials":[{"username":"'+ username + '"},{"password":"' + password + '"}]}'})
......@@ -72,9 +72,9 @@ class ICATParameterReader:
text.append('\n\n[ERROR] %s parameters are unknown by ICAT. Ingestion will fail.'%(str(len(missingParameters))))
else:
text.append('\n\n[ERROR] %s parameter is unknown by ICAT. Ingestion will fail.'%(str(len(missingParameters))))
self.LOG.info("Parameter list is not OK compared to ICAT database. missing=%s metadatamanager=%s"%(str(missingParameters), self.metadatamanager))
self.logger.info("Parameter list is not OK compared to ICAT database. missing=%s metadatamanager=%s"%(str(missingParameters), self.metadatamanager))
else:
self.LOG.info("Parameter list is OK compared to ICAT database. metadatamanager=%s", (self.metadatamanager))
self.logger.info("Parameter list is OK compared to ICAT database. metadatamanager=%s", (self.metadatamanager))
text.append("Parameter list is OK")
for parameter in missingParameters:
text.append('[Parameter] %s'%(parameter))
......
......@@ -49,9 +49,6 @@ class StompClient():
_JOLOKIA_USERNAME = 'user'
_JOLOKIA_PASSWORD = 'user'
# Logger
LOG = logging.getLogger("StompClient")
def __init__(self, queueURLs, queueName, beamlineID, manager=None, jolokia_port=None):
self.queueName = queueName
self.beamlineID = beamlineID
......@@ -65,20 +62,25 @@ class StompClient():
opener = urllib.request.build_opener(proxy_handler)
urllib.request.install_opener(opener)
if manager is None:
self.logger = logging.getLogger("StompClient")
else:
self.logger = logging.getLogger("StompClient({})".format(manager.get_name()))
# In case queueURLs is empty then messagingClient is set in status simulation
self.isSimulationMode = False
self._jolokia_port = jolokia_port
if len(queueURLs) == 1:
if queueURLs[0] == '':
self.isSimulationMode = True
self.LOG.info("Messaging client to ActiveMQ is in SIMULATION mode. beamline=%s" %(beamlineID))
self.logger.info("Messaging client to ActiveMQ is in SIMULATION mode. beamline=%s" %(beamlineID))
self.cfgURL = "simulator"
return
self.cfgURL = 'failover:(tcp://' + ',tcp://'.join(queueURLs) + ')'
self.cfgURL += '?maxReconnectAttempts=3,initialReconnectDelay=250,maxReconnectDelay=1000'
self.client = Stomp(StompConfig(self.cfgURL,version=StompSpec.VERSION_1_1))
self.LOG.info("Messaging client to ActiveMQ is connected. queueURLs=%s beamline=%s" % (queueURLs, beamlineID))
self.logger.info("Messaging client to ActiveMQ is connected. queueURLs=%s beamline=%s" % (queueURLs, beamlineID))
@property
def jolokia_port(self):
......@@ -117,7 +119,7 @@ class StompClient():
s = '%s processors running' % v
except Exception as e:
traceback.print_exc(file=sys.stdout)
self.LOG.error('Error getting Camel status: %s %s' % e, host)
self.logger.error('Error getting Camel status: %s %s' % e, host)
s = 'Processor status unknown'
status = "Connected to %s at %s. server=%s host=%s queueName=%s" % (self.client.session.server, self.client.session.server, host, host, self.queueName)
......@@ -145,12 +147,12 @@ class StompClient():
self.client.send(self.queueName, toBytes(msg), {'persistent':'true', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
except Exception as e:
traceback.print_exc()
self.LOG.error("Error sending message, trying again: %s beamline=%s " % (e,self.beamlineID))
self.logger.error("Error sending message, trying again: %s beamline=%s " % (e,self.beamlineID))
self.beat()
try:
self.client.send(self.queueName, toBytes(msg), {'persistent':'true', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
except Exception as e:
self.LOG.error("Unable to send message. error=%s beamline=%s " % (e,self.beamlineID))
self.logger.error("Unable to send message. error=%s beamline=%s " % (e,self.beamlineID))
def beat(self):
if self.isSimulationMode:
......@@ -166,13 +168,13 @@ class StompClient():
except Exception as e:
# if the heartbeat failed we close the connection
# this put the client in the correct DISCONNECTED state
self.LOG.warn("Missed a heartBeat. error=%s beamline=%s" % str(e.message, self.beamlineID))
self.logger.warn("Missed a heartBeat. error=%s beamline=%s" % str(e.message, self.beamlineID))
try:
self.client.close(True)
except:
pass
else:
self.LOG.info("Heart beat changed, will disconnect")
self.logger.info("Heart beat changed, will disconnect")
self.client.disconnect()
# after beat we call connect to ensure
# the connection is reset if it was closed
......@@ -185,7 +187,7 @@ class StompClient():
if self.isSimulationMode:
return
if self.client.session.state == StompClient.CONNECTED:
self.LOG.info("Disconnecting. metadataManagerId=%s beamline=%s" % (self._getManagerId(), self.beamlineID))
self.logger.info("Disconnecting. metadataManagerId=%s beamline=%s" % (self._getManagerId(), self.beamlineID))
if self.listener:
self.listener.stop()
self.client.close(True)
......@@ -200,24 +202,24 @@ class StompClient():
self.heartbeat = self.heartbeat_set
self.client.connect(versions=[StompSpec.VERSION_1_1],heartBeats=(self.heartbeat,self.heartbeat),connectedTimeout=1)
self.connected = True
self.LOG.info(self.getStatus().replace("\n", " - "))
self.logger.info(self.getStatus().replace("\n", " - "))
if self.heartbeat > 0:
self.LOG.info("Heartbeat to: %d" % self.heartbeat)
self.logger.info("Heartbeat to: %d" % self.heartbeat)
except Exception as e:
self.LOG.warn("Error connecting to ingestion queue. error=%s beamline=%s" % (e, self.beamlineID))
self.logger.warn("Error connecting to ingestion queue. error=%s beamline=%s" % (e, self.beamlineID))
traceback.print_exc()
self.connected = False
if self.connected and self.manager is not None:
try:
topic = '/topic/beamline.' + self.manager.beamlineID.lower()
#self.LOG.debug("Starting listener for %s. topic=%s beamline=%s" % (topic, topic, self.manager.beamlineID.lower()))
#self.logger.debug("Starting listener for %s. topic=%s beamline=%s" % (topic, topic, self.manager.beamlineID.lower()))
if self.listener:
self.listener.stop()
self.listener = _ListenerThread(self, topic, self.manager.beamlineID.lower())
self.listener.start()
except Exception as e:
traceback.print_exc()
self.LOG.warn("Error subscribing to %s: %s" % (topic, e))
self.logger.warn("Error subscribing to %s: %s" % (topic, e))
self.listener=None
def isConnected(self):
......@@ -237,9 +239,9 @@ class _ListenerThread(threading.Thread):
self.setDaemon(True)
self.parent = parent
subsId = self.parent.manager.getID()
self.parent.LOG.debug("Subscribing. subsId=%s queue=%s beamline=%s" % (str(subsId), str(queue), str(beamline)))
self.parent.logger.debug("Subscribing. subsId=%s queue=%s beamline=%s" % (str(subsId), str(queue), str(beamline)))
self.parent.client.subscribe(queue,{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL, StompSpec.ID_HEADER: subsId})
self.parent.LOG.info("Subscribed. subsId=%s queue=%s beamline=%s" % (subsId, queue, beamline))
self.parent.logger.info("Subscribed. subsId=%s queue=%s beamline=%s" % (subsId, queue, beamline))
def run(self):
while not self.__stop:
......@@ -252,7 +254,7 @@ class _ListenerThread(threading.Thread):
else:
time.sleep(0.5)
except Exception as e:
self.parent.LOG.warn("Error in listener %s" %e)
self.parent.logger.warn("Error in listener %s" %e)
try:
self.parent.beat()
except Exception:
......
......@@ -58,7 +58,6 @@ except ImportError:
class MetaExperiment (tango.LatestDeviceImpl):
"""Metadata for high level experiment"""
def __init__(self, cl, name):
tango.LatestDeviceImpl.__init__(self,cl,name)
......@@ -84,16 +83,16 @@ class MetaExperiment (tango.LatestDeviceImpl):
self.set_state(tango.DevState.ON)
# logger bridge
self.l = logging.getLogger()
self.l.setLevel(logging.DEBUG)
self.l.addHandler(TangoLoggingHandler(self))
logging.getLogger().setLevel(logging.DEBUG)
self.logger = logging.getLogger(self.get_name())
self.logger.addHandler(TangoLoggingHandler(self))
if graypy != None:
self.l.info("Init MetaExperiment logging. graylogServer=%s port=%s instrument=%s" % (self.graylogServer, self.graylogPort, self.beamlineID))
self.logger.info("Init MetaExperiment logging. graylogServer=%s port=%s instrument=%s" % (self.graylogServer, self.graylogPort, self.beamlineID))
try:
self.l.addHandler(graypy.GELFHandler(self.graylogServer, self.graylogPort))
self.logger.addHandler(graypy.GELFHandler(self.graylogServer, self.graylogPort))
except:
try:
self.l.addHandler(graypy.GELFUDPHandler(self.graylogServer, self.graylogPort))
self.logger.addHandler(graypy.GELFUDPHandler(self.graylogServer, self.graylogPort))
except:
traceback.print_exc(file=sys.stdout)
......
This diff is collapsed.
......@@ -37,7 +37,6 @@ class MetadataWorker():
'''
Class containing the login of the metadata engine
'''
LOG = logging.getLogger("MetadataWorker")
# Allowed chars for folder and file names
validFilenameChars = ".-_%s%s" % (string.ascii_letters, string.digits)
......@@ -46,6 +45,7 @@ class MetadataWorker():
'''
Constructor
'''
self.logger = logging.getLogger("MetadataWorker({})".format(metadataManagerName))
self.entries = []
self.writer = HDF5FileWriter(metadataManagerName)
self.fileExplorer = FileExplorer()
......@@ -131,9 +131,9 @@ class MetadataWorker():
params = self.toDictionary(dataset, sampleParamMap, initial=True)
# export single dataset file
self.LOG.debug(self.entries)
self.logger.debug(self.entries)
self.entries[metadatafilename] = self.writer.appendTo(metadatafilename, params, HDF5FileWriter.PHASE_INITIAL, [])
self.LOG.debug(self.entries)
self.logger.debug(self.entries)
# export to all global files
......@@ -141,10 +141,10 @@ class MetadataWorker():
if HDF5FileWriter.ENTRY_KEY in params:
del params[HDF5FileWriter.ENTRY_KEY]
try:
self.LOG.warn("Exporting parameters to master. file=%s metadatamanager=%s" %(f, self.metadataManagerName))
self.logger.warn("Exporting parameters to master. file=%s metadatamanager=%s" %(f, self.metadataManagerName))
entries[f] = self.writer.appendTo(f, params, HDF5FileWriter.PHASE_INITIAL, [])
except Exception as e:
self.LOG.warn("Error exporting %s: %s. metadatamanager=%s" %(f, e, self.metadataManagerName))
self.logger.warn("Error exporting %s: %s. metadatamanager=%s" %(f, e, self.metadataManagerName))
except Exception as e:
traceback.print_exc(file=sys.stdout)
return self.entries
......@@ -170,9 +170,9 @@ class MetadataWorker():
# export single dataset file
if entries[metadatafilename] is not None:
self.LOG.info("Exporting HDF5. metadatafilename=%s metadatamanager=%s" %(str(entries[metadatafilename]), self.metadataManagerName))
self.logger.info("Exporting HDF5. metadatafilename=%s metadatamanager=%s" %(str(entries[metadatafilename]), self.metadataManagerName))
else:
self.LOG.info("Exporting HDF5 metadatafilename is None. metadatamanager=%s" %(self.metadataManagerName))
self.logger.info("Exporting HDF5 metadatafilename is None. metadatamanager=%s" %(self.metadataManagerName))
params[HDF5FileWriter.ENTRY_KEY] = entries[metadatafilename]
try:
......@@ -181,7 +181,7 @@ class MetadataWorker():
self.writer.appendTo(metadatafilename, params, HDF5FileWriter.PHASE_FINAL, outputFiles)
except Exception as e:
self.LOG.warn("Error exporting. metadatafilename=%s error=%s metadatamanager=%s" %(metadatafilename, e, self.metadataManagerName))
self.logger.warn("Error exporting. metadatafilename=%s error=%s metadatamanager=%s" %(metadatafilename, e, self.metadataManagerName))
# export to all global files
for f in files:
if f in entries:
......@@ -189,9 +189,9 @@ class MetadataWorker():
try:
entries[f] = self.writer.appendTo(f, params, HDF5FileWriter.PHASE_FINAL, outputFiles)
except Exception as e:
self.LOG.warn("Error exporting. file=%s error=%s metadatamanager=%s" %(f, e, self.metadataManagerName))
self.logger.warn("Error exporting. file=%s error=%s metadatamanager=%s" %(f, e, self.metadataManagerName))
else:
self.LOG.warn("Not trying to export to %s, entry not registered. metadatamanager=%s" % (f, self.metadataManagerName))
self.logger.warn("Not trying to export to %s, entry not registered. metadatamanager=%s" % (f, self.metadataManagerName))
except Exception as e:
traceback.print_exc(file=sys.stdout)
......@@ -221,7 +221,7 @@ class MetadataWorker():
x = 0
xname = fname
while os.path.isfile(xname):
self.LOG.warn("File already exists, will use an incremented file name. file=%s metadatamanager=%s" % (xname, self.metadataManagerName))
self.logger.warn("File already exists, will use an incremented file name. file=%s metadatamanager=%s" % (xname, self.metadataManagerName))
xname = fname[:-(len(ext))] + '-' + str(x) + ext
x += 1
return xname
......@@ -243,9 +243,9 @@ class MetadataWorker():
f = open(filepath, 'w-')
# write the file
dataset.export(f, 0, pretty_print=True)
self.LOG.info("Dataset exported as XML. filepath=%s metadatamanager=%s" % (filepath, self.metadataManagerName))
self.logger.info("Dataset exported as XML. filepath=%s metadatamanager=%s" % (filepath, self.metadataManagerName))
except Exception as e:
self.LOG.error("Error writing metadata. file=%s error=%s metadatamanager=%s" % (filepath, e, self.metadataManagerName))
self.logger.error("Error writing metadata. file=%s error=%s metadatamanager=%s" % (filepath, e, self.metadataManagerName))
finally:
if f:
try:
......
......@@ -20,7 +20,7 @@ class ParameterDefinition():
REMOTE = 'remote'
MULTI = 'multi'
LOG = logging.getLogger("ParameterDefinition")
logger = logging.getLogger("ParameterDefinition")
def __init__(self, parameterString):
'''
......@@ -104,7 +104,7 @@ class ParameterDefinition():
for p in self.remoteList:
s = 'ERROR'
try:
self.LOG.debug('Remote attribute %s' % p[0])
self.logger.debug('Remote attribute %s' % p[0])
attribute = attributeProxyMap[p[0]]
if attribute:
......@@ -142,7 +142,7 @@ class ParameterDefinition():
else:
s = str(json.loads(valueMap[p[0]])[p[1]])
except Exception as e:
self.LOG.error("Error building attribute %s: %s" %(p[0], e))
self.logger.error("Error building attribute %s: %s" %(p[0], e))
strrep.append(s)
return ' '.join(strrep)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment