Commit 3d107ee8 authored by myron's avatar myron
Browse files

Merge branch 'alessandro_develop'

parents adab0a7f 55940064
test3:
stage: test
only:
- scisoft12
tags:
- scisoft12
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python3/
- virtualenv --python=python3 ~/software/virtualenv_xrstools_python3/
- source ~/software/virtualenv_xrstools_python3/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- pip install -r requirements.txt
- ln -s /usr/lib/python3/dist-packages/PyQt* ~/software//virtualenv_xrstools_python3/lib/python3.4/site-packages/
- ln -s /usr/lib/python3/dist-packages/sip* ~/software//virtualenv_xrstools_python3/lib/python3.4/site-packages/
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- deactivate
after_script:
- pwd
stages:
- test
# - deploy
variables:
PROJECT_NAME: "xrstools"
LOCAL_REPO: "/dev/shm/gitlab_ci/xrstools"
VENV_DIR: /dev/shm/gitlab_ci/xrstools/venv
USE_CLEAN_VENV: "false"
CLEAN_RESULTS: "true"
test:
stage: test
only:
- scisoft13
- nocdi
tags:
- scisoft13
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python2/
- virtualenv --python=python2 ~/software/virtualenv_xrstools_python2/
- source ~/software/virtualenv_xrstools_python2/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- scisoft10
before_script:
- unset HTTP_PROXY
- unset HTTPS_PROXY
- if [ ! -d $LOCAL_REPO ]; then mkdir -p $LOCAL_REPO; fi
- if [ -d $VENV_DIR ]; then if [ "$USE_CLEAN_VENV" == "true" ]; then rm -rf $VENV_DIR; fi; fi
- python3 -m venv $VENV_DIR
- source $VENV_DIR/bin/activate
- echo "Running $(python --version) from $(which python) on $(hostname) ($(arch))"
- pip install --upgrade pip setuptools wheel
- pip install --upgrade numpy silx
- pip install PyQt5
- pip install -r requirements.txt
- ln -s /usr/lib/python2.7/dist-packages/PyQt* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- ln -s /usr/lib/python2.7/dist-packages/sip* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- deactivate
after_script:
- pwd
testnslino:
stage: test
only:
- nslino
tags:
- nslino
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python2/
- virtualenv --python=python2 ~/software/virtualenv_xrstools_python2/
- source ~/software/virtualenv_xrstools_python2/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- pip install -r requirements.txt
- ln -s /usr/lib/python2.7/dist-packages/PyQt* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- ln -s /usr/lib/python2.7/dist-packages/sip* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- cd nonregressions/xes
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python xes_analysis.py >| output
- cd ../xrs_raman
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py >| output1
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py >| output2
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py >| output3
- cd ../volumes/test1
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python test.py >| output
- deactivate
after_script:
- pwd
test3nslino:
stage: test
only:
- nslino3
tags:
- nslino
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python3/
- virtualenv --python=python3 ~/software/virtualenv_xrstools_python3/
- source ~/software/virtualenv_xrstools_python3/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- pip install -r requirements.txt
- pip install PyQt5
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- deactivate
after_script:
- pwd
\ No newline at end of file
[submodule "HighFive"]
path = fitcc/HighFive
url = https://github.com/BlueBrain/HighFive.git
import os
import json
import h5py
import numpy as np
def runit( s , work_dir, filename ) :
open( os.path.join(work_dir, filename) ,"w" ).write(s)
os.system( "cd %s; XRS_swissknife %s ; cd .." % (work_dir, filename) )
def prepare(
work_dir="preparation_run4_16",
data = "/data/id20/inhouse/data/run4_16/run6_ihr/rixs" ,
scan_for_roi= 66 ,
do_roi = True,
do_roi_sample = True,
reference_scan = 66 , # the scan used for the reference
monitorcolumn = "izero",
do_recentering=False,
scan_for_sample_roi = 268 , # Not necessary is recentering is not needed. If not given the rois file is assumed (when needed if needed) to be already there
beta_response = 0
) :
if not os.path.exists(work_dir):
os.makedirs(work_dir)
assert (os.path.exists( work_dir) and os.path.isdir( work_dir) ), "Cannot get work directory %s "%work_dir
assert (os.access(work_dir, os.W_OK | os.X_OK)), " directory %s is not writable"%work_dir
preparation_parameters = {
"work_dir" : work_dir,
"data" : data,
"scan_for_roi" : scan_for_roi,
"reference_scan" : reference_scan,
"monitorcolumn" : monitorcolumn,
"do_recentering" : do_recentering,
"scan_for_sample_roi" : scan_for_sample_roi
}
json.dump(preparation_parameters, open(os.path.join(work_dir, "preparation_parameters.json"),"w") )
if do_roi:
s="""
create_rois:
expdata : "{data_file}"
scans : [{scan_for_roi}]
roiaddress : "roi.h5:/ROI_AS_SELECTED"
""".format( data_file = data, scan_for_roi = scan_for_roi )
runit( s , work_dir, "01_create_roi.yaml" )
if do_recentering and do_roi_sample:
s="""
create_rois:
expdata : "{data_file}"
scans : [{scan_for_roi} ]
roiaddress : roi_sample.h5:/ROI_AS_SELECTED
""".format( data_file = data, scan_for_roi = scan_for_sample_roi )
runit( s , work_dir, "02_create_roi_sample.yaml" )
s="""
loadscan_2Dimages :
expdata : "{data_file}"
roiaddress : roi.h5:/ROI_AS_SELECTED
scan_interval : [{reference_scan},{reference_scan_P1}]
signaladdress : calibration_scan
sumto1D : 0
energycolumn : 'Anal Energy'
monitorcolumn : {monitorcolumn}
save_also_roi : 1
""".format( data_file = data, reference_scan = reference_scan , reference_scan_P1 = reference_scan +1 , monitorcolumn = monitorcolumn )
runit( s , work_dir, "03_load_scan_reference.yaml" )
if do_recentering :
s="""
loadscan_2Dimages :
expdata : "{data_file}"
roiaddress : roi_sample.h5:/ROI_AS_SELECTED
scan_interval : [{scan_for_roi},{scan_for_roi_P1}]
signaladdress : uncentered_sample_scan
sumto1D : 0
energycolumn : 'Anal Energy'
monitorcolumn : {monitorcolumn}
save_also_roi : 1
""".format( data_file = data, scan_for_roi = scan_for_sample_roi , scan_for_roi_P1 = scan_for_sample_roi +1 , monitorcolumn = monitorcolumn )
runit( s , work_dir, "04_load_scan_sample.yaml" )
s="""
calculate_recenterings :
bariA : roi_sample.h5:/ROI_AS_SELECTED/uncentered_sample_scan/scans/Scan%03d/
bariB : roi.h5:/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d/
target : recentering.h5:/recentering
"""%( scan_for_sample_roi, reference_scan )
runit( s , work_dir, "05_setup_recentering.yaml" )
s="""
loadscan_2Dimages :
expdata : {data_file}
roiaddress : roi.h5:/ROI_AS_SELECTED
scan_interval : [{scan_for_roi},{scan_for_roi_P1}]
monitorcolumn : {monitorcolumn}
recenterings : recentering.h5:/recentering
recenterings_confirmed : recentering.h5:/confirmed_shift
signaladdress : Centered_sample_scans
sumto1D : 0
energycolumn : 'Anal Energy'
""".format( data_file = data, scan_for_roi = scan_for_sample_roi , scan_for_roi_P1 = scan_for_sample_roi +1 , monitorcolumn = monitorcolumn )
runit( s , work_dir, "06_calculate_recentering.yaml" )
s="""
superR_fit_responses:
foil_scan_address : roi.h5:/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d
nref : 2
niter_optical : 50
beta_optical : %e
niter_global : 2
pixel_dim : 1
niter_pixel : 0
beta_pixel : 0
do_refine_trajectory : 1
simmetrizza : 1
filter_rois : 0
target_file : fitted_response.h5:/ScanFittedResponse_%03d
fit_lines : 1
"""%( reference_scan, beta_response, reference_scan )
runit( s , work_dir, "07_fit.yaml" )
s="""
superR_recreate_rois :
responsefilename : fitted_response.h5:/ScanFittedResponse_%03d
nex : 0
old_scan_address : roi.h5:/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d
filter_rois : 0
recenterings_refined : recentering.h5:/confirmed_shift
target_filename : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_%03d
"""%( reference_scan, reference_scan, reference_scan )
runit( s , work_dir, "08_resynth.yaml" )
def extract(
work_dir="preparation_run4_16",
scan_interval = [130, 132] ,
target_file = "SPECTRA.h5",
temporary_file = "temporary.h5",
niter = 0,
niterLip = 20,
beta = 0,
) :
assert (os.path.exists( work_dir) and os.path.isdir( work_dir) ), "Cannot get work directory %s "%work_dir
assert (os.access(work_dir, os.W_OK | os.X_OK)), " directory %s is not writable"%work_dir
preparation_parameters = json.load( open(os.path.join(work_dir, "preparation_parameters.json"),"r") )
data = preparation_parameters["data"]
reference_scan = preparation_parameters["reference_scan"]
monitorcolumn = preparation_parameters["monitorcolumn"]
scan_for_roi = preparation_parameters["scan_for_roi"]
start = scan_interval[0]
end = scan_interval[1]
s="""
loadscan_2Dimages :
expdata : {data_file}
roiaddress : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_{reference_scan:03d}
scan_interval : [{start}, {end}]
signaladdress : ../{temporary_file}:/data_{start}_{end}
sumto1D : 0
energycolumn : 'Anal Energy'
monitorcolumn : {monitorcolumn}
""".format(data_file = data, reference_scan = reference_scan , monitorcolumn = monitorcolumn,
start = start, end = end, temporary_file=temporary_file)
runit( s , work_dir, "31_harvest.yaml" )
s="""
extract_spectra :
reference_address : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_{reference_scan:03d}
sample_address : ../{temporary_file}:/data_{start}_{end}
roiaddress : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_{reference_scan:03d}
reference_scan : {reference_scan}
scan_interval : [{start}, {end}]
zmargin : 0
target : ../{result_file}:/fromscans_{start}_{end}
# final_plot : PLOT
DE : 0.0
niterLip : {niterLip}
niter : {niter}
beta : {beta}
""".format(reference_scan=reference_scan, start = start, end = end , niterLip = niterLip, niter = niter, beta = beta, result_file = target_file, temporary_file=temporary_file)
runit( s , work_dir, "32_extract_spectra.yaml" )
......@@ -2,6 +2,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import scipy
import json
### __doc__="""
......@@ -126,13 +127,19 @@ def filterRoiList(l, strip=False, prefix="ROI"):
result = [ str(int(stripROI(t))) for t in l if ( t not in [ "motorDict", "response"] and t[:len(prefix)]==prefix )]
else:
result = [ t for t in l if ( t not in [ "motorDict", "response"] and t[:len(prefix)]==prefix ) ]
result.sort()
result = sorted( result , key = lambda x: int(''.join(filter(str.isdigit, str(x) ))) )
### result.sort()
return result
def filterScanList(l, prefix="Scan"):
result = [ t for t in l if ( t not in [ "motorDict", "response"] and t[:len(prefix)]==prefix ) ]
result.sort()
result = sorted( result , key = lambda x: int(''.join(filter(str.isdigit, str(x) ))) )
# result.sort()
return result
......@@ -190,6 +197,12 @@ from yaml.constructor import Constructor, MappingNode, ConstructorError
from XRStools import fit_spectra
from XRStools import reponse_percussionelle
def check_allowed_keys( mydata, allowed_keys ) :
for k in mydata.keys():
if not k in allowed_keys:
raise ValueError( (" key "+str(k) +" not in allowed keys :"+ str(allowed_keys)) )
def dump_anydict_as_map( anydict):
yaml.add_representer( anydict, _represent_dictorder)
......@@ -230,16 +243,15 @@ class Loader_map_as_anydict( object):
class myOrderedDict (collections.OrderedDict):
def __setitem__(self,a,b):
## print "cucu",a,b
if type(a)==type("") and a in self:
self[a+"_tagkajs"]=b
self[a+"_TAG_this_key_is_given_twice"]=b
else:
## print super(myOrderedDict, self)
super(myOrderedDict, self).__setitem__(a,b )
def cleaned(key):
while key[-8:]=="_tagkajs":
while key[-28:]=="_TAG_this_key_is_given_twice":
key=key[:-8]
return key
......@@ -300,7 +312,6 @@ def main():
mydata = yamlData[key]
if isinstance(mydata,dict) and "active" in mydata :
if mydata["active"]==0:
# print " continuo "
continue
if key != "help":
......@@ -351,81 +362,6 @@ def help(yamlData):
print( func.__doc__)
def Extraction(mydata):
"""
**Extraction**
Function to extract the interesting signal after removal of Compton profile,
linear baselines, Pearson profile
Example ::
Extraction :
active : 1
dataadress : "pippo.h5:/ROI_A/loaded_datas" # where load_scans wrote data
HFaddress : "pippo.h5:/ROI_A/loaded_datas/HF_O" # where compton profiles have been calculated
# prenormrange : [ 5 , .inf ]
analyzerAverage : # averaging over analysers
active : 1
which : [0,11 , 36,59 ]
errorweighing : False
removeLinearAv : # fit a linear baseline and remove it
active : 1
region1 : [520.0,532.0]
region2 : None
ewindow : 100
scale : 1
removePearsonAv: # fit a Pearson and remove it
active : 0
region1 : [520.0,532.0]
region2 : None
guess :
Peak_position : 600.0
FWHM : 10
Shape : "Lorentzian"
Peak_intensity: 100.0
linear_slope : 1
linear_background : 0
scaling_factor : 1
view : 0
target : "myextraction" # path relative to dataadress where extracted signal will be written
"""
reader , filename, groupname= read_reader(mydata, name="dataadress")
HF = read_HF(mydata, name="hfspectrum_address")
extr = extraction.extraction(reader , HF)
if ("analyzerAverage") in mydata :
aa_data = mydata["analyzerAverage"]
if gvord(aa_data,"active",True):
which = aa_data["which"]
errorweighing = gvord(aa_data,"errorweighing",False)
extr .analyzerAverage(which,errorweighing=errorweighing)
if ("removeLinearAv") in mydata :
rla_data = mydata["removeLinearAv"]
if gvord(rla_data,"active",True):
region1 = rla_data["region1"]
region2 = gvord( rla_data,"region2",None)
ewindow = gvord( rla_data,"ewindow",100)
scale = gvord( rla_data,"scale",100)
extr .removeLinearAv(region1, region2=region2,ewindow=ewindow,
scale=scale, view = gvord(mydata,"view",False),
)
print( gvord(mydata,"view",False))
groupname = groupname+"/"+ mydata["target"]
check_libre( filename , groupname )
extr.save_state_hdf5( filename,groupname, comment = inputtext )
def split_hdf5_address(dataadress):
......@@ -439,233 +375,188 @@ but : was not found
return filename, groupname
def read_HF(mydata, name="hfspectrum_address"):
dataadress = mydata[name]
filename, groupname = split_hdf5_address(dataadress)
HF = theory.HFspectrum(None,None,None, initialise=False)
HF.load_state_hdf5( filename, groupname)
return HF
def HFspectrum(mydata):
"""
**HFspectrum**
function for building S(q,w) from tabulated Hartree-Fock Compton profiles to use in
the extraction algorithm.
EXAMPLE ::
dataadress : "hdf5filename:full_nameof_signals_group" # where load_scans wrote data
formulas : ['O'] # list of strings of chemical sum formulas of which the sample is made up
concentrations : [1.0] # list of concentrations of how the different chemical formulas are mixed (sum should be 1)
correctasym : [[0.0,0.0,0.0]] # single value or list of scaling values for the HR-correction to
# the 1s, 2s, and 2p shells. one value per element in the list of formulas
hfspectrum_address : "nameofgroup" # Target group for writing Relative to dataadress (and in the same file)!!!!
"""
reader , filename, groupname= read_reader(mydata, name="dataadress")
hf = theory.HFspectrum(reader ,
mydata["formulas"] ,
mydata["concentrations"] ,
mydata["correctasym"]
)
groupname = groupname+"/"+ mydata["hfspectrum_address"]
check_libre( filename , groupname )
hf.save_state_hdf5( filename,groupname , comment = inputtext )
# def load_scans(mydata):
# """
# **load_scans**
def load_scans(mydata):
"""
**load_scans**
This command harvest the selected signals.
the instructions on the scans to be taken must be in the form( as submembers ofload_scans ) ::
# This command harvest the selected signals.
# the instructions on the scans to be taken must be in the form( as submembers ofload_scans ) ::
load_scans :
roiaddress : "hdf5filename:nameofroigroup" # the same given in create_rois
expdata : "absolutepathtoaspecfile" # this points to a spec file
# load_scans :
# roiaddress : "hdf5filename:nameofroigroup" # the same given in create_rois
# expdata : "absolutepathtoaspecfile" # this points to a spec file
elastic_scans : [623]
fine_scans : [626,630,634,638,642]
n_loop : 4
long_scan : 624
# elastic_scans : [623]
# fine_scans : [626,630,634,638,642]
# n_loop : 4
# long_scan : 624
signaladdress : "nameofsignalgroup" # Target group for writing Relative to ROI (and in the same file)!!!!
# signaladdress : "nameofsignalgroup" # Target group for writing Relative to ROI (and in the same file)!!!!
#############################################################
# OPTIONALS
#
order : [0,1,2,3,4,5] # list of integers (0-5) which describes the order of modules in which the
# ROIs were defined (default is VD, VU, VB, HR, HL, HB; i.e. [0,1,2,3,4,5])
# #############################################################
# # OPTIONALS
# #
# order : [0,1,2,3,4,5] # list of integers (0-5) which describes the order of modules in which the