Commit 7ad8bb5b authored by Alejandro De Maria Antolinos's avatar Alejandro De Maria Antolinos

Merge branch 'issue_35' into 'master'

Issue 35

See merge request demariaa/tango-metadata!1
parents 97c53645 6a6cfc08
*sh
*~
.metadata
target/*
......@@ -7,12 +8,18 @@ target/*
.project
.pydevproject
*.pyc
*.egg-info
activemq-data/
log/
data/
build/
export.h5
icat_ingest_metadata
stompest
roberto
problem
*.org
src/main
dist
.cache
.idea
venv
# Metadata Manager and Experiment
1. [Dataset Location](#dataset-location)
* [DataFolderPattern](#datafolderpattern)
* [Specific dataset Location](#specific-dataset-location)
2. [Requirements](#requirements)
3. [Configuration](#configuration)
* [Tango](#tango)
* [ICAT Reader](#icat-reader)
* [ICAT+](#icat+)
4. [Logging with Graylog](#logging-with-graylog)
5. [Build](#build)
* [Packaging](#packaging)
* [Deploy](#deploy)
5. [Elogbook](#elogbook)
* [Notifications](#notifications)
- [Metadata Manager and Experiment](#Metadata-Manager-and-Experiment)
- [Dataset Location](#Dataset-Location)
- [DataFolderPattern](#DataFolderPattern)
- [Specific dataset Location](#Specific-dataset-Location)
- [Requirements](#Requirements)
- [Debian 6](#Debian-6)
- [Configuration](#Configuration)
- [Tango](#Tango)
- [Using automatic tool to generate tango properties](#Using-automatic-tool-to-generate-tango-properties)
- [ICAT Reader](#ICAT-Reader)
- [ICAT+](#ICAT)
- [Logging with Graylog](#Logging-with-Graylog)
- [Build](#Build)
- [Packaging](#Packaging)
- [Deploy](#Deploy)
- [Elogbook](#Elogbook)
- [Requirements](#Requirements-1)
- [Usage](#Usage)
- [Python](#Python)
- [Simulation](#Simulation)
7. [Simulation](#simulation)
## Dataset Location
First Metadata Manager needs to be configured with some parameters. Next parameters are mandatory:
- **dataFolderPattern**: pattern that will be replaced by the values from parameters to determine the dataset location. For instance: {dataRoot}/{proposal}/{beamlineID}/{sampleName}/{scanName}
- **globalHDFfiles**: patters that will be replaced by the values from parameters to determine the location where global HDF5 file will be stored. For instance: {dataRoot}/{proposal}/{beamlineID}/{proposal}-{beamlineID}.h5
- **globalHDFfiles**: patterns that will be replaced by the values from parameters to determine the location where global HDF5 file will be stored. For instance: {dataRoot}/{proposal}/{beamlineID}/{proposal}-{beamlineID}.h5
- **beamlineId**: name of the beamline. For instance: id30a3, bm31
To be Changed!!
......@@ -127,7 +133,7 @@ GenerateMetaDataResources.py -b id99
The use in Jive -> File/Load property file to load the generated file.
Example of generated files:
```
MetadataManager_ID99.tango
MetaExperiment_ID99.tango
```
......@@ -335,7 +341,10 @@ if __name__ == '__main__':
```
## Simulation
MetaExperiment and MetadataManager can be run in Simulation mode. It means that it will not try to access to the ActiveMQ server.
In order to activate the simulation mode someone just need to empty the queueURLs parameter
![icat-properties.png](documentation/classStatic.png)
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>fr.esrf.icat</groupId>
<artifactId>icat-super-pom</artifactId>
<version>1.8-SNAPSHOT</version>
</parent>
<artifactId>MetadataManager</artifactId>
<version>3.9</version>
<packaging>pom</packaging>
<name>Tango MetadataManager</name>
<scm>
<developerConnection>scm:git:git@gitlab.esrf.fr:icat/tango-metadata.git</developerConnection>
<connection>scm:git:http://gitlab.esrf.fr/icat/tango-metadata.git</connection>
<url>http://gitlab.esrf.fr/icat/tango-metadata.git</url>
<tag>HEAD</tag>
</scm>
<properties>
<python.root.folder>${project.build.directory}/python-archiver</python.root.folder>
<python.shade.folder>${project.build.directory}/python-shader</python.shade.folder>
<metadata.artifact>icat-ingest-metadata</metadata.artifact>
<metadata.version>1.6</metadata.version>
<stomp.package>stompest-2.1.4</stomp.package>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>prepare-package</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${python.root.folder}</outputDirectory>
<escapeString>\</escapeString>
<resources>
<resource>
<directory>src/main/python</directory>
<filtering>false</filtering>
<excludes><exclude>setup.py</exclude></excludes>
</resource>
<resource>
<directory>src/main/python</directory>
<filtering>true</filtering>
<includes><include>setup.py</include></includes>
</resource>
<resource>
<directory>${python.shade.folder}/${python.metadata.artifact}-${python.metadata.version}/${python.metadata.artifact}</directory>
<filtering>false</filtering>
<includes><include>*.py</include></includes>
<targetPath>${python.root.folder}/${python.package.name}/${python.metadata.artifact}</targetPath>
</resource>
<resource>
<directory>${python.shade.folder}/${stomp.package}/stompest</directory>
<filtering>false</filtering>
<includes><include>**</include></includes>
<targetPath>${python.root.folder}/${python.package.name}/stompest</targetPath>
</resource>
<resource>
<directory>${project.basedir}/src/main/scripts</directory>
<filtering>true</filtering>
<includes><include>**</include></includes>
<targetPath>${python.shade.folder}/scripts</targetPath>
</resource>
<resource>
<directory>${project.basedir}/src/main/scripts</directory>
<filtering>false</filtering>
<includes><include>**</include></includes>
<targetPath>scripts/</targetPath>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>prepare</id>
<phase>compile</phase>
<configuration>
<target>
<untar src="${project.basedir}/src/assembly/${stomp.package}.tar.gz" dest="${python.shade.folder}" compression="gzip" />
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>unpack-python</id>
<phase>compile</phase>
<goals><goal>unpack</goal></goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>fr.esrf.icat</groupId>
<artifactId>${metadata.artifact}</artifactId>
<version>${metadata.version}</version>
<type>zip</type>
<classifier>python</classifier>
<overWrite>true</overWrite>
<outputDirectory>${python.shade.folder}</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${basedir}/src/assembly/distrib.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-python-distrib</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>pythonVersion</id>
<goals><goal>regex-property</goal></goals>
<phase>validate</phase>
<configuration>
<name>python.version</name>
<regex>-SNAPSHOT</regex>
<value>${project.version}</value>
<replacement>\.preview</replacement>
<failIfNoMatch>false</failIfNoMatch>
</configuration>
</execution>
<execution>
<id>pythonMetadataVersion</id>
<goals><goal>regex-property</goal></goals>
<phase>validate</phase>
<configuration>
<name>python.metadata.version</name>
<regex>-SNAPSHOT</regex>
<value>${metadata.version}</value>
<replacement>\.preview</replacement>
<failIfNoMatch>false</failIfNoMatch>
</configuration>
</execution>
<execution>
<id>pythonPackageName</id>
<goals><goal>regex-property</goal></goals>
<phase>validate</phase>
<configuration>
<name>python.package.name</name>
<regex>-</regex>
<value>${project.artifactId}</value>
<replacement>_</replacement>
<failIfNoMatch>false</failIfNoMatch>
</configuration>
</execution>
<execution>
<id>pythonMetadataArtifact</id>
<goals><goal>regex-property</goal></goals>
<phase>validate</phase>
<configuration>
<name>python.metadata.artifact</name>
<regex>-</regex>
<value>${metadata.artifact}</value>
<replacement>_</replacement>
<failIfNoMatch>false</failIfNoMatch>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
......@@ -30,8 +30,8 @@ MetadataManager/{BL}/DEVICE/MetadataManager: "{BL}/metadata/mgr"
# --- {BL}/metadata/mgr properties
{BL}/metadata/mgr->beamlineID: {BL}
{BL}/metadata/mgr->dataFolderPattern: "{{dataRoot}}/{{sampleName}}/{{scanName}}"
{BL}/metadata/mgr->globalHDFfiles: "{{dataRoot}}/{{scanName}}.h5"
{BL}/metadata/mgr->dataFolderPattern: "{{dataRoot}}/{{sampleName}}/{{datasetName}}"
{BL}/metadata/mgr->globalHDFfiles: "{{dataRoot}}/{{datasetName}}.h5"
{BL}/metadata/mgr->labels: "definition = test"
{BL}/metadata/mgr->metaExperimentDevice: "{BL}/metadata/exp"
{BL}/metadata/mgr->parameters: InstrumentSource_mode
......@@ -42,7 +42,7 @@ MetadataManager/{BL}/DEVICE/MetadataManager: "{BL}/metadata/mgr"
{BL}/metadata/mgr/dataFolder->__value: ""
{BL}/metadata/mgr/InstrumentSource_mode->__value: bunch_later
{BL}/metadata/mgr/metadataFile->__value: ""
{BL}/metadata/mgr/scanName->__value: ""
{BL}/metadata/mgr/datasetName->__value: ""
#---------------------------------------------------------
# CLASS MetadataManager properties
......@@ -94,9 +94,9 @@ CLASS/MetaExperiment->queueURLs: bcu-mq-01.esrf.fr:61613
def main(args):
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
print " MetaData TANGO DEVICE SERVER RESOURCES GENERATOR "
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n"
print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
print(" MetaData TANGO DEVICE SERVER RESOURCES GENERATOR ")
print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n")
parser = OptionParser(description = 'MetaData TANGO DEVICE SERVER RESOURCES GENERATOR')
parser.add_option('-d', action = "store_true", default = False, help = 'debug', metavar = 'DEBUG')
......@@ -116,7 +116,7 @@ def main(args):
debug = options.d
if debug: print >> sys.stderr, "Options:", options
if debug: print("Options:", options, file=sys.stderr)
try:
# Opens a file named according to BL number.
......@@ -128,13 +128,13 @@ def main(args):
sys.exc_info()[2])
rf.close()
else:
print "Tango resources for {0} MetaData will be generated in file {1}".format(options.bl_name, res_file_name)
print("Tango resources for {0} MetaData will be generated in file {1}".format(options.bl_name, res_file_name))
rf.write(Resources.format(BL=bl_name, date=time.asctime(time.localtime(time.time())),program=os.path.basename(sys.argv[0])))
rf.close()
print
print()
if __name__ == "__main__":
main(sys.argv)
......
......@@ -4,67 +4,67 @@ import time
d1 = PyTango.DeviceProxy("test/meta/1")
d2 = PyTango.DeviceProxy("test/ingest/1")
print "device: " + d1.name()
print "device: " + d2.name()
print(d2.get_property('parameters'))
print(d2.get_attribute_list())
print("device: " + d1.name())
print("device: " + d2.name())
print((d2.get_property('parameters')))
print((d2.get_attribute_list()))
time.sleep(5)
print "Resetting ..."
print("Resetting ...")
d2.Reset()
print("State: " + str(d2.status()) + " - " + str(d2.state()))
print(("State: " + str(d2.status()) + " - " + str(d2.state())))
time.sleep(5)
print "New proposal 'test0'"
print("New proposal 'test0'")
d1.proposal = "test0"
print "Proposal: " + d2["proposal"].value
print("State: " + str(d2.status()) + " - " + str(d2.state()))
print("Proposal: " + d2["proposal"].value)
print(("State: " + str(d2.status()) + " - " + str(d2.state())))
time.sleep(5)
print "New Sample 'sample1'"
print("New Sample 'sample1'")
d1.sample = "sample1"
print "Sample: " + d2["sampleName"].value
print("Sample: " + d2["sampleName"].value)
time.sleep(5)
print "Set sample parameters"
print("Set sample parameters")
d2.SetSampleParameters(["key1","value1","key2","value2"])
print(d2["sampleParameters"].value)
print((d2["sampleParameters"].value))
time.sleep(5)
d2.SetSampleParameters(["key1","newvalue!!","key3","value3"])
print(d2["sampleParameters"].value)
print((d2["sampleParameters"].value))
time.sleep(5)
print "New Scan 'scan1'"
d2.scanName = "scan1"
print("State: " + str(d2.status()) + " - " + str(d2.state()))
print("Proposed data folder " + d2["dataFolder"].value)
print("Proposed metadata file " + d2["metadataFile"].value)
print("New Dataset 'dataset1'")
d2.datasetName = "dataset1"
print(("State: " + str(d2.status()) + " - " + str(d2.state())))
print(("Proposed data folder " + d2["dataFolder"].value))
print(("Proposed metadata file " + d2["metadataFile"].value))
time.sleep(5)
print "direct parameter set"
print("direct parameter set")
time.sleep(5)
d2.filter = "my filter"
time.sleep(5)
print "Start scan"
d2.StartScan()
print("State: " + str(d2.status()) + " - " + str(d2.state()))
print("Start dataset")
d2.StartDataset()
print(("State: " + str(d2.status()) + " - " + str(d2.state())))
time.sleep(5)
print "direct parameter set"
print("direct parameter set")
time.sleep(5)
d2.filter = "my new filter"
time.sleep(5)
print "recording files"
print("recording files")
time.sleep(5)
d2.lastDatafile = "file1"
time.sleep(5)
d2.lastDatafile= "file2"
time.sleep(5)
print "Ending scan"
d2.EndScan()
print("State: " + str(d2.status()) + " - " + str(d2.state()))
print("Ending dataset")
d2.EndDataset()
print(("State: " + str(d2.status()) + " - " + str(d2.state())))
time.sleep(5)
#!/usr/bin/env python
from setuptools import setup, find_packages
setup(
name='MetadataManager',
version='1.5.4',
packages=find_packages(),
install_requires=[],
include_package_data=True
)
\ No newline at end of file
version='4.0.0',
description="ESRF Data Policy metadata manager and meta experiment Tango servers",
# license="",
# author="",
# url="https://gitlab.esrf.fr/icat/tango-metadata",
packages=["metadata_manager", "metadata_manager.icat_ingest_metadata"],
package_dir={"":"src", "metadata_manager.icat_ingest_metadata": "src/metadata_manager/icat_ingest_metadata" },
package_data={"metadata_manager": ["*.xml"]},
entry_points={"console_scripts": [
"MetadataManager = metadata_manager.MetadataManager:main",
"MetaExperiment = metadata_manager.MetaExperiment:main",
]},
install_requires=[
"stompest",
"h5py",
],
)
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>distrib</id>
<formats>
<format>zip</format>
<format>tar.gz</format>
</formats>
<baseDirectory>${python.package.name}-${python.version}</baseDirectory>
<fileSets>
<fileSet>
<outputDirectory>/</outputDirectory>
<directory>${python.root.folder}</directory>
</fileSet>
<fileSet>
<outputDirectory>/data/</outputDirectory>
<directory>src/assembly</directory>
<excludes>
<exclude>**/*</exclude>
</excludes>
</fileSet>
</fileSets>
<files>
<file>
<source>${python.shade.folder}/scripts/MetadataManager.sh</source>
<outputDirectory>/bin</outputDirectory>
<destName>MetadataManager</destName>
<filtered>false</filtered>
<lineEnding>unix</lineEnding>
<fileMode>0744</fileMode>
</file>
<file>
<source>${python.shade.folder}/scripts/MetaExperiment.sh</source>
<outputDirectory>/bin</outputDirectory>
<destName>MetaExperiment</destName>
<filtered>false</filtered>
<lineEnding>unix</lineEnding>
<fileMode>0744</fileMode>
</file>
<file>
<source>${python.shade.folder}/scripts/MetadataExport.sh</source>
<outputDirectory>/bin</outputDirectory>
<destName>MetadataExport</destName>
<filtered>false</filtered>
<lineEnding>unix</lineEnding>
<fileMode>0744</fileMode>
</file>
</files>
</assembly>
\ No newline at end of file
'''
Created on Mar 26, 2015
@author: cleva
'''
import logging
import icat
import icat.config
from HDF5FileWriter import HDF5FileWriter
logging.basicConfig(level=logging.INFO)
config = icat.config.Config()
config.add_variable('file', ("-o", "--outputfile"),
dict(help="output file name"),
default='export.h5')
config.add_variable('investigation', ("investigation",),
dict(help="name and optionally visit id "
"(separated by a colon) of the investigation"))
conf = config.getconfig()
client = icat.Client(conf.url, **conf.client_kwargs)
print "Connected to ICAT version %s" % client.apiversion
client.login(conf.auth, conf.credentials)
inv = client.search("Investigation [name='%s']" % conf.investigation)[0]
print "Located Proposal %s" % inv.name
writer = HDF5FileWriter()
datasets = client.search("Dataset INCLUDE Sample,DatasetParameter,ParameterType [investigation.id=%s]" % inv.id)
print "Located %s Datasets for the Proposal" % len(datasets)
for dts in datasets:
d = dict()
d['proposal'] = inv.name
d['beamlineID'] = inv.visitId
d['scanName'] = dts.name
d['location'] = dts.location
d['startDate'] = dts.startDate
d['endDate'] = dts.endDate
d['sampleName'] = dts.sample.name
for p in dts.parameters:
if p.numericValue is not None:
d[p.type.name] = p.numericValue
elif p.dateTimeValue is not None:
d[p.type.name] = p.dateTimeValue
else:
d[p.type.name] = p.stringValue
writer.appendTo(conf.file, d, 'initial')
writer.appendTo(conf.file, d, 'final')
print "Proposal %s exported in %s" %(inv.name, conf.file)
<?xml version="1.0" encoding="ASCII"?>
<pogoDsl:PogoSystem xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:pogoDsl="http://www.esrf.fr/tango/pogo/PogoDsl">
<classes name="MetaExperiment" pogoRevision="9.0">
<description description="Metadata for high level experiment" title="Metadata for high level experiment" sourcePath="/users/cleva/workspaces/metadataPyTango/tango-metadata/src/main/python/MetadataManager" language="Python" filestogenerate="XMI file,Code files" license="GPL" hasMandatoryProperty="true" hasConcreteProperty="true" hasAbstractCommand="false" hasAbstractAttribute="false">
<inheritances classname="Device_Impl" sourcePath=""/>
<identification contact="at esrf.fr - andy.gotz" author="andy.gotz" emailDomain="esrf.fr" classFamily="SoftwareSystem" siteSpecific="ESRF" platform="All Platforms" bus="Not Applicable" manufacturer="none" reference=""/>
</description>
<classProperties name="queueURLs" description="URLs of the queues (including port)">
<type xsi:type="pogoDsl:StringVectorType"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<DefaultPropValue>&quot;bcu-mq-01.esrf.fr:61613&quot;</DefaultPropValue>
</classProperties>
<classProperties name="queueName" description="Name of the stomp queue for sending ingestion requests">
<type xsi:type="pogoDsl:StringType"/>
<status abstract="false" inherited="false" concrete="true" concreteHere="true"/>
<DefaultPropValue>/queue/icatIngest</DefaultPropValue>
</classProperties>