Build new release

parent 66ce74fb
<?xml version="1.0" encoding="UTF-8" ?>
<project name="maven-antrun-" default="main" >
<target name="main">
<untar src="/users/demariaa/Software/workspaces/icat/tango-metadata/src/assembly/stompest-2.1.4.tar.gz" dest="/users/demariaa/Software/workspaces/icat/tango-metadata/target/python-shader" compression="gzip"/>
</target>
</project>
\ No newline at end of file
#! /bin/bash
. blissrc
BASE_FOLDER=${BLISSADM}/applications/MetadataManager
APPLICATION=MetadataManager/MetaExperiment.py
cd ${BASE_FOLDER}
exec python ${APPLICATION} "$@"
\ No newline at end of file
#! /bin/bash
. blissrc
BASE_FOLDER=${BLISSADM}/applications/MetadataManager
APPLICATION=MetadataManager/MetaExperiment.py
cd ${BASE_FOLDER}
exec python ${APPLICATION} "$@"
\ No newline at end of file
#! /bin/bash
. blissrc
BASE_FOLDER=${BLISSADM}/applications/MetadataManager
APPLICATION=MetadataManager/HDF5Export.py
cd ${BASE_FOLDER}
exec python ${APPLICATION} -w https://ovm-icat.esrf.fr:8181/ICATService/ICAT?wsdl --no-proxy .esrf.fr,localhost -a esrf -u $1 -P -o data/$3 $2
\ No newline at end of file
#! /bin/bash
. blissrc
BASE_FOLDER=${BLISSADM}/applications/MetadataManager
APPLICATION=MetadataManager/HDF5Export.py
cd ${BASE_FOLDER}
exec python ${APPLICATION} -w https://ovm-icat.esrf.fr:8181/ICATService/ICAT?wsdl --no-proxy .esrf.fr,localhost -a esrf -u $1 -P -o data/$3 $2
\ No newline at end of file
#! /bin/bash
. blissrc
BASE_FOLDER=${BLISSADM}/applications/MetadataManager
APPLICATION=MetadataManager/MetadataManager.py
cd ${BASE_FOLDER}
exec python ${APPLICATION} "$@"
\ No newline at end of file
#! /bin/bash
. blissrc
BASE_FOLDER=${BLISSADM}/applications/MetadataManager
APPLICATION=MetadataManager/MetadataManager.py
cd ${BASE_FOLDER}
exec python ${APPLICATION} "$@"
\ No newline at end of file
# ##########################################################################
#
# Copyright (c) 2016 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
import os
import logging
class FileExplorer():
'''
Class for exploring and getting the files from the outputfolder
Filters will be surely implemented in the future
'''
def __init__(self):
'''
Constructor
'''
def getFiles(self, folderPath):
self.folderPath = folderPath
list = []
for dirname, dirnames, filenames in os.walk(folderPath):
for subdirname in dirnames:
print "FOUND DIRECTORY: ", os.path.join(dirname, subdirname)
for filename in filenames:
list.append(os.path.join(dirname, filename))
return list
'''
Created on Mar 26, 2015
@author: cleva
'''
import logging
import icat
import icat.config
from HDF5FileWriter import HDF5FileWriter
logging.basicConfig(level=logging.INFO)
config = icat.config.Config()
config.add_variable('file', ("-o", "--outputfile"),
dict(help="output file name"),
default='export.h5')
config.add_variable('investigation', ("investigation",),
dict(help="name and optionally visit id "
"(separated by a colon) of the investigation"))
conf = config.getconfig()
client = icat.Client(conf.url, **conf.client_kwargs)
print "Connected to ICAT version %s" % client.apiversion
client.login(conf.auth, conf.credentials)
inv = client.search("Investigation [name='%s']" % conf.investigation)[0]
print "Located Proposal %s" % inv.name
writer = HDF5FileWriter()
datasets = client.search("Dataset INCLUDE Sample,DatasetParameter,ParameterType [investigation.id=%s]" % inv.id)
print "Located %s Datasets for the Proposal" % len(datasets)
for dts in datasets:
d = dict()
d['proposal'] = inv.name
d['beamlineID'] = inv.visitId
d['scanName'] = dts.name
d['location'] = dts.location
d['startDate'] = dts.startDate
d['endDate'] = dts.endDate
d['sampleName'] = dts.sample.name
for p in dts.parameters:
if p.numericValue is not None:
d[p.type.name] = p.numericValue
elif p.dateTimeValue is not None:
d[p.type.name] = p.dateTimeValue
else:
d[p.type.name] = p.stringValue
writer.appendTo(conf.file, d, 'initial')
writer.appendTo(conf.file, d, 'final')
print "Proposal %s exported in %s" %(inv.name, conf.file)
This diff is collapsed.
'''
Created on Oct 28, 2014
@author: cleva
'''
from stompest.config import StompConfig
from stompest.protocol import StompSession
from stompest.protocol import StompSpec
from stompest.sync import Stomp
from datetime import datetime
from StringIO import StringIO
import logging
import threading
import time
import urllib2
import base64
import json
class StompClient():
'''
classdocs
'''
# Stomp client session state
CONNECTING = StompSession.CONNECTING
CONNECTED = StompSession.CONNECTED
DISCONNECTING = StompSession.DISCONNECTING
DISCONNECTED = StompSession.DISCONNECTED
_TIMEOUT = 0.1
JOLOKIA_CONSUMER_COUNT_URL = 'api/jolokia/read/org.apache.activemq:type=Broker,brokerName=metadata,destinationType=Queue,destinationName=icatIngest/ConsumerCount'
JOLOKIA_PORT = '8778'
_JOLOKIA_USERNAME = 'user'
_JOLOKIA_PASSWORD = 'user'
# Logger
LOG = logging.getLogger("StompClient")
def __init__(self, queueURLs, queueName, manager=None):
'''
Constructor
'''
self.queueName = queueName
self.heartbeat = 0
self.heartbeat_set = 0
self.connected = False
self.manager = manager
self.cfgURL = 'failover:(tcp://' + ',tcp://'.join(queueURLs) + ')'
self.cfgURL += '?maxReconnectAttempts=3,initialReconnectDelay=250,maxReconnectDelay=1000'
self.client = Stomp(StompConfig(self.cfgURL,version=StompSpec.VERSION_1_1))
self.listener=None
proxy_handler = urllib2.ProxyHandler({})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
def getConfigURL(self):
return self.cfgURL
def getState(self):
return self.client.session.state
def getStatus(self):
if self.connected:
host = self.client._transport.host
try:
request = urllib2.Request("http://%s:%s/%s" %(host, self.JOLOKIA_PORT, self.JOLOKIA_CONSUMER_COUNT_URL))
request.add_header('Authorization', b'Basic ' + base64.b64encode(self._JOLOKIA_USERNAME + b':' + self._JOLOKIA_PASSWORD))
result = urllib2.urlopen(request, timeout=1)
jres = json.load(result)
result.close()
v = int(jres['value'])
if v < 1:
s = 'No processor running'
else:
s = '%s processors running' % v
except Exception as e:
self.LOG.error('Error getting Camel status: %s %s' % e, host)
s = 'Processor status unknown'
status = "Connected to %s at %s" % (self.client.session.server, host)
if s is not None:
status = status + "\n" + s
return status
else:
return "Not connected to any server"
def sendObject(self, xmlObj):
xmlS = StringIO()
xmlObj.export(xmlS, 0, pretty_print=False)
self.sendMessage(xmlS.getvalue())
xmlS.close()
def sendMessage(self, msg):
# sends the msg in the queue
# tries to reconnect after a 1st failed attempt
# for failover to work
try:
self.client.send(self.queueName, msg, {'persistent':'true', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
except Exception as e:
self.LOG.warn("Error sending message, trying again: %s" % e)
self.beat()
try:
self.client.send(self.queueName, msg, {'persistent':'true', StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
except Exception as e:
self.LOG.error("Unable to send message: %s" % e)
def beat(self):
# does a heartbeat if connected
# maintaining a connection requires to call beat() at
# a frequency below the hearbeat
if self.client.session.state == StompSession.CONNECTED:
# if the heartbeat changed, we disconnect
if self.heartbeat == self.heartbeat_set:
try:
self.LOG.debug("Beating ...")
self.client.beat()
except Exception as e:
# if the heartbeat failed we close the connection
# this put the client in the correct DISCONNECTED state
self.LOG.warn("Missed a heartBeat: %s" %e)
try:
self.client.close(True)
except:
pass
else:
self.LOG.info("Heart beat changed, will disconnect")
self.client.disconnect()
# after beat we call connect to ensure
# the connection is reset if it was closed
self.connect()
def setHeartBeat(self, beat):
self.heartbeat_set = beat
def disconnect(self):
if self.client.session.state == StompClient.CONNECTED:
self.LOG.info("Disconnecting ...")
if self.listener:
self.listener.stop()
self.client.close(True)
self.connected = False
def connect(self):
# if we disconnected or never connected, try to connect
if self.client.session.state == StompSession.DISCONNECTED:
try:
self.heartbeat = self.heartbeat_set
self.client.connect(versions=[StompSpec.VERSION_1_1],heartBeats=(self.heartbeat,self.heartbeat),connectedTimeout=1)
self.connected = True
self.LOG.info(self.getStatus().replace("\n", " - "))
if self.heartbeat > 0:
self.LOG.info("Heartbeat to: %d" % self.heartbeat)
except Exception as e:
self.LOG.warn("Error connecting to ingestion queue: %s" %e)
self.connected = False
if self.connected and self.manager is not None:
try:
topic = '/topic/beamline.' + self.manager.beamlineID.lower()
self.LOG.debug("Starting listener for %s" % topic)
if self.listener:
self.listener.stop()
self.listener = _ListenerThread(self, topic)
self.listener.start()
except Exception as e:
self.LOG.warn("Error subscribing to %s: %s" % (topic, e))
self.listener=None
def isConnected(self):
return self.connected
def onReceive(self, frame):
if self.manager is not None:
self.manager.appendMessage("%s: %s" %(datetime.now().replace(microsecond=0), frame.body))
class _ListenerThread(threading.Thread):
def __init__(self, parent, queue):
threading.Thread.__init__(self)
self.__stop = False
self.setDaemon(True)
self.parent = parent
self.parent.LOG.debug("Subscribing to %s" % queue)
subsId = self.parent.manager.getID()
self.parent.client.subscribe(queue,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL, StompSpec.ID_HEADER: subsId})
self.parent.LOG.info("Subscribed to %s as %s" % (queue, subsId))
def run(self):
while not self.__stop:
try:
if self.parent.isConnected():
if self.parent.client.canRead(timeout=0.5):
frame = self.parent.client.receiveFrame()
self.parent.onReceive(frame)
self.parent.client.ack(frame)
else:
time.sleep(0.5)
except Exception as e:
self.parent.LOG.warn("Error in listener %s" %e)
try:
self.parent.beat()
except Exception:
pass
time.sleep(0.5)
def stop(self, wait=False):
self.__stop = True
if wait:
self.join()
This diff is collapsed.
<?xml version="1.0" encoding="ASCII"?>
<pogoDsl:PogoSystem xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:pogoDsl="http://www.esrf.fr/tango/pogo/PogoDsl">
<classes name="MetaExperiment" pogoRevision="9.0">
<description description="Metadata for high level experiment" title="Metadata for high level experiment" sourcePath="/users/cleva/workspaces/metadataPyTango/tango-metadata/src/main/python/MetadataManager" language="Python" filestogenerate="XMI file,Code files" license="GPL" hasMandatoryProperty="true" hasConcreteProperty="true" hasAbstractCommand="false" hasAbstractAttribute="false">
<inheritances classname="Device_Impl" sourcePath=""/>
<identification contact="at esrf.fr - andy.gotz" author="andy.gotz" emailDomain="esrf.fr" classFamily="SoftwareSystem" siteSpecific="ESRF" platform="All Platforms" bus="Not Applicable" manufacturer="none" reference=""/>
</description>
<classProperties name="queueURLs" description="URLs of the queues (including port)">
<type xsi:type="pogoDsl:StringVectorType"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<DefaultPropValue>&quot;bcu-mq-01.esrf.fr:61613&quot;</DefaultPropValue>
</classProperties>
<classProperties name="queueName" description="Name of the stomp queue for sending ingestion requests">
<type xsi:type="pogoDsl:StringType"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<DefaultPropValue>/queue/icatIngest</DefaultPropValue>
</classProperties>
<deviceProperties name="beamlineID" mandatory="true" description="Beamline ID for reporting metadata">
<type xsi:type="pogoDsl:StringType"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
</deviceProperties>
<commands name="State" description="This command gets the device state (stored in its device_state data member) and returns it to the caller." execMethod="dev_state" displayLevel="OPERATOR" polledPeriod="0" isDynamic="false">
<argin description="none">
<type xsi:type="pogoDsl:VoidType"/>
</argin>
<argout description="Device state">
<type xsi:type="pogoDsl:StateType"/>
</argout>
<status abstract="true" inherited="true" concrete="true" concreteHere="true"/>
</commands>
<commands name="Status" description="This command gets the device status (stored in its device_status data member) and returns it to the caller." execMethod="dev_status" displayLevel="OPERATOR" polledPeriod="0">
<argin description="none">
<type xsi:type="pogoDsl:VoidType"/>
</argin>
<argout description="Device status">
<type xsi:type="pogoDsl:ConstStringType"/>
</argout>
<status abstract="true" inherited="true" concrete="true"/>
</commands>
<attributes name="proposal" attType="Scalar" rwType="READ_WRITE" displayLevel="OPERATOR" polledPeriod="0" maxX="" maxY="" memorized="true" allocReadMember="false" isDynamic="false">
<dataType xsi:type="pogoDsl:StringType"/>
<changeEvent fire="false" libCheckCriteria="false"/>
<archiveEvent fire="false" libCheckCriteria="false"/>
<dataReadyEvent fire="false" libCheckCriteria="true"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<properties description="" label="" unit="" standardUnit="" displayUnit="" format="" maxValue="" minValue="" maxAlarm="" minAlarm="" maxWarning="" minWarning="" deltaTime="" deltaValue=""/>
</attributes>
<attributes name="sample" attType="Scalar" rwType="READ_WRITE" displayLevel="OPERATOR" polledPeriod="0" maxX="" maxY="" memorized="true" allocReadMember="false" isDynamic="false">
<dataType xsi:type="pogoDsl:StringType"/>
<changeEvent fire="false" libCheckCriteria="false"/>
<archiveEvent fire="false" libCheckCriteria="false"/>
<dataReadyEvent fire="false" libCheckCriteria="true"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<properties description="" label="" unit="" standardUnit="" displayUnit="" format="" maxValue="" minValue="" maxAlarm="" minAlarm="" maxWarning="" minWarning="" deltaTime="" deltaValue=""/>
</attributes>
<attributes name="dataRoot" attType="Scalar" rwType="READ_WRITE" displayLevel="EXPERT" polledPeriod="0" maxX="" maxY="" memorized="true" allocReadMember="true" isDynamic="false">
<dataType xsi:type="pogoDsl:StringType"/>
<changeEvent fire="false" libCheckCriteria="false"/>
<archiveEvent fire="false" libCheckCriteria="false"/>
<dataReadyEvent fire="false" libCheckCriteria="true"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<properties description="" label="" unit="" standardUnit="" displayUnit="" format="" maxValue="" minValue="" maxAlarm="" minAlarm="" maxWarning="" minWarning="" deltaTime="" deltaValue=""/>
</attributes>
<states name="ON" description="Everything OK">
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
</states>
<preferences docHome="./doc_html" makefileHome="/segfs/tango/cppserver/env"/>
</classes>
</pogoDsl:PogoSystem>
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
'''
Created on Nov 7, 2014
@author: cleva
'''
import re
import logging
import json
import sys, traceback
import numpy
class ParameterDefinition():
'''
classdocs
'''
LIST_PARAMETER_REGEX = re.compile('\A\s*\[\s*[a-zA-Z0-9_\-]+(?:\s*,\s*[a-zA-Z0-9_\-]+)*\s*\]\s*\Z')
REMOTE_PARAMETER_REG = re.compile('\A\s*([a-zA-Z0-9\.:_\-/]+)(?:\[([a-zA-Z0-9_\-/]+)\])?\s*\Z')
SIMPLE = 'simple'
LIST = 'list'
REMOTE = 'remote'
MULTI = 'multi'
LOG = logging.getLogger("ParameterDefinition")
def __init__(self, parameterString):
'''
Constructor
'''
self.optionList = None
self.remoteList = None
self.originalString = parameterString
# name = definition
d = parameterString.split("=")
self.parameterName = d[0].strip()
# SIMPLE (no definition)
if len(d) == 1:
self.parameterDefinition = None
self.parameterType = self.SIMPLE
# LIST (list of options)
elif self.LIST_PARAMETER_REGEX.match(d[1]):
self.parameterDefinition = d[1].strip()
self.parameterType = self.LIST
self.optionList = []
for s in self.parameterDefinition[1:-1].split(","):
self.optionList.append(s.strip())
# REMOTE (remote attribute(s))
else:
self.parameterDefinition = d[1].strip()
self.remoteList = []
# list of REMOTES
r = self.parameterDefinition.split(",")
if(len(r) > 1):
self.parameterType = self.MULTI
else:
self.parameterType = self.REMOTE
# build list of pairs (attribute, index)
# index is None if absent
for s in r:
m = self.REMOTE_PARAMETER_REG.match(s)
g = m.groups()
# if index is digits, convert to int
# list index vs. dict key
# prevents using all digit strings as dict key
if g[1] and g[1].isdigit():
t = g[0], int(g[1])
else:
t = g[0], g[1]
self.remoteList.append(t)
def getOriginalString(self):
return self.originalString
def getParameterName(self):
return self.parameterName
def getParameterDefinition(self):
return self.parameterDefinition
def getParameterType(self):
return self.parameterType
def getOptionList(self):
return self.optionList
def getRemoteList(self):
return self.remoteList
def getRemoteAttributes(self):
if self.remoteList is None:
return None
return map(lambda p: p[0], self.remoteList)
def isLocal(self):
return self.parameterType == self.SIMPLE or self.parameterType == self.LIST
def isRemote(self):
return self.parameterType == self.REMOTE or self.parameterType == self.MULTI
def getValue(self, attributeProxyMap):
strrep = []
self.LOG.debug('Reading attribute %s' % self.parameterName)
for p in self.remoteList:
s = 'ERROR'
try: