Commit 3d107ee8 authored by myron's avatar myron
Browse files

Merge branch 'alessandro_develop'

parents adab0a7f 55940064
test3:
stage: test
only:
- scisoft12
tags:
- scisoft12
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python3/
- virtualenv --python=python3 ~/software/virtualenv_xrstools_python3/
- source ~/software/virtualenv_xrstools_python3/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- pip install -r requirements.txt
- ln -s /usr/lib/python3/dist-packages/PyQt* ~/software//virtualenv_xrstools_python3/lib/python3.4/site-packages/
- ln -s /usr/lib/python3/dist-packages/sip* ~/software//virtualenv_xrstools_python3/lib/python3.4/site-packages/
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- deactivate
stages:
- test
# - deploy
after_script:
- pwd
variables:
PROJECT_NAME: "xrstools"
LOCAL_REPO: "/dev/shm/gitlab_ci/xrstools"
VENV_DIR: /dev/shm/gitlab_ci/xrstools/venv
USE_CLEAN_VENV: "false"
CLEAN_RESULTS: "true"
test:
stage: test
only:
- scisoft13
- nocdi
tags:
- scisoft13
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python2/
- virtualenv --python=python2 ~/software/virtualenv_xrstools_python2/
- source ~/software/virtualenv_xrstools_python2/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- scisoft10
before_script:
- unset HTTP_PROXY
- unset HTTPS_PROXY
- if [ ! -d $LOCAL_REPO ]; then mkdir -p $LOCAL_REPO; fi
- if [ -d $VENV_DIR ]; then if [ "$USE_CLEAN_VENV" == "true" ]; then rm -rf $VENV_DIR; fi; fi
- python3 -m venv $VENV_DIR
- source $VENV_DIR/bin/activate
- echo "Running $(python --version) from $(which python) on $(hostname) ($(arch))"
- pip install --upgrade pip setuptools wheel
- pip install --upgrade numpy silx
- pip install PyQt5
- pip install -r requirements.txt
- ln -s /usr/lib/python2.7/dist-packages/PyQt* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- ln -s /usr/lib/python2.7/dist-packages/sip* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- deactivate
after_script:
- pwd
testnslino:
stage: test
only:
- nslino
tags:
- nslino
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python2/
- virtualenv --python=python2 ~/software/virtualenv_xrstools_python2/
- source ~/software/virtualenv_xrstools_python2/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- pip install -r requirements.txt
- ln -s /usr/lib/python2.7/dist-packages/PyQt* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- ln -s /usr/lib/python2.7/dist-packages/sip* ~/software//virtualenv_xrstools_python2/lib/python2.7/site-packages/
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- cd nonregressions/xes
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python xes_analysis.py >| output
- cd ../xrs_raman
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py >| output1
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py >| output2
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py >| output3
- cd ../volumes/test1
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python test.py >| output
- deactivate
after_script:
- pwd
\ No newline at end of file
test3nslino:
stage: test
only:
- nslino3
tags:
- nslino
script:
- echo 'current dir is '
- pwd
- unset PYTHONPATH
- export PATH=/usr/local/bin:/usr/bin:/bin:/opt/oar/utilities:/sbin:/usr/bin/X11:.::.
- unset LDFLAGS
- unset CPPFLAGS
- unset LD_LIBRARY_PATH
- rm -rf ~/software/virtualenv_xrstools_python3/
- virtualenv --python=python3 ~/software/virtualenv_xrstools_python3/
- source ~/software/virtualenv_xrstools_python3/bin/activate
- pip install setuptools --upgrade
- pip install pip --upgrade
- pip install numpy
- pip install -r requirements.txt
- pip install PyQt5
- pip install .
- cd nonregressions
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_small.py
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a python non_reg_testing_XRS_raman_extraction.py
- deactivate
after_script:
- pwd
[submodule "HighFive"]
path = fitcc/HighFive
url = https://github.com/BlueBrain/HighFive.git
import os
import json
import h5py
import numpy as np
def runit( s , work_dir, filename ) :
open( os.path.join(work_dir, filename) ,"w" ).write(s)
os.system( "cd %s; XRS_swissknife %s ; cd .." % (work_dir, filename) )
def prepare(
work_dir="preparation_run4_16",
data = "/data/id20/inhouse/data/run4_16/run6_ihr/rixs" ,
scan_for_roi= 66 ,
do_roi = True,
do_roi_sample = True,
reference_scan = 66 , # the scan used for the reference
monitorcolumn = "izero",
do_recentering=False,
scan_for_sample_roi = 268 , # Not necessary is recentering is not needed. If not given the rois file is assumed (when needed if needed) to be already there
beta_response = 0
) :
if not os.path.exists(work_dir):
os.makedirs(work_dir)
assert (os.path.exists( work_dir) and os.path.isdir( work_dir) ), "Cannot get work directory %s "%work_dir
assert (os.access(work_dir, os.W_OK | os.X_OK)), " directory %s is not writable"%work_dir
preparation_parameters = {
"work_dir" : work_dir,
"data" : data,
"scan_for_roi" : scan_for_roi,
"reference_scan" : reference_scan,
"monitorcolumn" : monitorcolumn,
"do_recentering" : do_recentering,
"scan_for_sample_roi" : scan_for_sample_roi
}
json.dump(preparation_parameters, open(os.path.join(work_dir, "preparation_parameters.json"),"w") )
if do_roi:
s="""
create_rois:
expdata : "{data_file}"
scans : [{scan_for_roi}]
roiaddress : "roi.h5:/ROI_AS_SELECTED"
""".format( data_file = data, scan_for_roi = scan_for_roi )
runit( s , work_dir, "01_create_roi.yaml" )
if do_recentering and do_roi_sample:
s="""
create_rois:
expdata : "{data_file}"
scans : [{scan_for_roi} ]
roiaddress : roi_sample.h5:/ROI_AS_SELECTED
""".format( data_file = data, scan_for_roi = scan_for_sample_roi )
runit( s , work_dir, "02_create_roi_sample.yaml" )
s="""
loadscan_2Dimages :
expdata : "{data_file}"
roiaddress : roi.h5:/ROI_AS_SELECTED
scan_interval : [{reference_scan},{reference_scan_P1}]
signaladdress : calibration_scan
sumto1D : 0
energycolumn : 'Anal Energy'
monitorcolumn : {monitorcolumn}
save_also_roi : 1
""".format( data_file = data, reference_scan = reference_scan , reference_scan_P1 = reference_scan +1 , monitorcolumn = monitorcolumn )
runit( s , work_dir, "03_load_scan_reference.yaml" )
if do_recentering :
s="""
loadscan_2Dimages :
expdata : "{data_file}"
roiaddress : roi_sample.h5:/ROI_AS_SELECTED
scan_interval : [{scan_for_roi},{scan_for_roi_P1}]
signaladdress : uncentered_sample_scan
sumto1D : 0
energycolumn : 'Anal Energy'
monitorcolumn : {monitorcolumn}
save_also_roi : 1
""".format( data_file = data, scan_for_roi = scan_for_sample_roi , scan_for_roi_P1 = scan_for_sample_roi +1 , monitorcolumn = monitorcolumn )
runit( s , work_dir, "04_load_scan_sample.yaml" )
s="""
calculate_recenterings :
bariA : roi_sample.h5:/ROI_AS_SELECTED/uncentered_sample_scan/scans/Scan%03d/
bariB : roi.h5:/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d/
target : recentering.h5:/recentering
"""%( scan_for_sample_roi, reference_scan )
runit( s , work_dir, "05_setup_recentering.yaml" )
s="""
loadscan_2Dimages :
expdata : {data_file}
roiaddress : roi.h5:/ROI_AS_SELECTED
scan_interval : [{scan_for_roi},{scan_for_roi_P1}]
monitorcolumn : {monitorcolumn}
recenterings : recentering.h5:/recentering
recenterings_confirmed : recentering.h5:/confirmed_shift
signaladdress : Centered_sample_scans
sumto1D : 0
energycolumn : 'Anal Energy'
""".format( data_file = data, scan_for_roi = scan_for_sample_roi , scan_for_roi_P1 = scan_for_sample_roi +1 , monitorcolumn = monitorcolumn )
runit( s , work_dir, "06_calculate_recentering.yaml" )
s="""
superR_fit_responses:
foil_scan_address : roi.h5:/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d
nref : 2
niter_optical : 50
beta_optical : %e
niter_global : 2
pixel_dim : 1
niter_pixel : 0
beta_pixel : 0
do_refine_trajectory : 1
simmetrizza : 1
filter_rois : 0
target_file : fitted_response.h5:/ScanFittedResponse_%03d
fit_lines : 1
"""%( reference_scan, beta_response, reference_scan )
runit( s , work_dir, "07_fit.yaml" )
s="""
superR_recreate_rois :
responsefilename : fitted_response.h5:/ScanFittedResponse_%03d
nex : 0
old_scan_address : roi.h5:/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d
filter_rois : 0
recenterings_refined : recentering.h5:/confirmed_shift
target_filename : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_%03d
"""%( reference_scan, reference_scan, reference_scan )
runit( s , work_dir, "08_resynth.yaml" )
def extract(
work_dir="preparation_run4_16",
scan_interval = [130, 132] ,
target_file = "SPECTRA.h5",
temporary_file = "temporary.h5",
niter = 0,
niterLip = 20,
beta = 0,
) :
assert (os.path.exists( work_dir) and os.path.isdir( work_dir) ), "Cannot get work directory %s "%work_dir
assert (os.access(work_dir, os.W_OK | os.X_OK)), " directory %s is not writable"%work_dir
preparation_parameters = json.load( open(os.path.join(work_dir, "preparation_parameters.json"),"r") )
data = preparation_parameters["data"]
reference_scan = preparation_parameters["reference_scan"]
monitorcolumn = preparation_parameters["monitorcolumn"]
scan_for_roi = preparation_parameters["scan_for_roi"]
start = scan_interval[0]
end = scan_interval[1]
s="""
loadscan_2Dimages :
expdata : {data_file}
roiaddress : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_{reference_scan:03d}
scan_interval : [{start}, {end}]
signaladdress : ../{temporary_file}:/data_{start}_{end}
sumto1D : 0
energycolumn : 'Anal Energy'
monitorcolumn : {monitorcolumn}
""".format(data_file = data, reference_scan = reference_scan , monitorcolumn = monitorcolumn,
start = start, end = end, temporary_file=temporary_file)
runit( s , work_dir, "31_harvest.yaml" )
s="""
extract_spectra :
reference_address : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_{reference_scan:03d}
sample_address : ../{temporary_file}:/data_{start}_{end}
roiaddress : newscan.h5:/ROI_AS_SELECTED/calibration_scan/scans/ScanReSynth_{reference_scan:03d}
reference_scan : {reference_scan}
scan_interval : [{start}, {end}]
zmargin : 0
target : ../{result_file}:/fromscans_{start}_{end}
# final_plot : PLOT
DE : 0.0
niterLip : {niterLip}
niter : {niter}
beta : {beta}
""".format(reference_scan=reference_scan, start = start, end = end , niterLip = niterLip, niter = niter, beta = beta, result_file = target_file, temporary_file=temporary_file)
runit( s , work_dir, "32_extract_spectra.yaml" )
This diff is collapsed.
......@@ -36,7 +36,7 @@ def Fista(solution , problem, niter, niterLip):
x_old = solution
errList=[]
for i in range(1000):
for i in range(niter):
grad, err = problem.calculate_grad(y)
print( "err " , err)
errList.append(err)
......
......@@ -46,8 +46,8 @@ class MaskImageWidget(sole_MaskImageWidget.MaskImageWidget):
self.plotImage(update = False)
self._emitMaskChangedSignal()
return
# super(MaskImageWidget, self)._graphSignal(ddict, ownsignal)
sole_MaskImageWidget.MaskImageWidget._graphSignal(self, ddict, ownsignal)
super(MaskImageWidget, self)._graphSignal(ddict, ownsignal)
# sole_MaskImageWidget.MaskImageWidget._graphSignal(self, ddict, ownsignal)
def dragEnterEvent(self,event):
......
......@@ -685,7 +685,7 @@ class plotContainer(QtGui.QWidget) :
if len(regionNorm):
norm = np.trapz(data[regionNorm,1],data[regionNorm,0])
print(" NOOOOOOOOOOOOOOOOOOOOOOOOOOOORM ", norm)
print(" NORM ", norm)
data[:,1] /= norm
data[:,2] /= norm
np.savetxt(prefix+"_"+self.name+".txt",data)
......
......@@ -6,6 +6,7 @@ import math
import scipy
import scipy.stats
import scipy.ndimage.filters as filt
from scipy.ndimage import gaussian_filter
import sys
import pickle
import os
......@@ -377,7 +378,7 @@ def Fista( data , solution , s2d, d2s, solution_shape , parallel = 0 , ni
if myrank==0:
print( indent+"CALCULATING LIPSCHITZ FACTOR ")
for i in range(100):
for i in range(10):
calculate_grad(grad2,None ,grad , s2d, d2s, solution_shape , parallel = parallel , beta=beta)
Lip = math.sqrt( np.linalg.norm(grad2/100000) )*100000
grad[:] = grad2/ Lip
......@@ -726,7 +727,10 @@ def get_spots_list( filename , groupname , filter_rois =1 ):
xscales = {}
enescan = None
for sn in filterRoiList(list(h5[groupname].keys())):
tmp_list = list(filterRoiList(list(h5[groupname].keys())))
tmp_list = sorted(tmp_list, key = int )
for sn in tmp_list:
print( groupname+"/"+sn+"/matrix")
m = h5[groupname+"/"+sn+"/matrix"][:]
......@@ -740,7 +744,7 @@ def get_spots_list( filename , groupname , filter_rois =1 ):
if groupname+"/motorDict/energy" in h5:
enescan = h5[groupname+"/motorDict/energy"].value
enescan = h5[groupname+"/motorDict/energy"][()]
if m.shape!=(0,):
......@@ -1010,7 +1014,6 @@ def DOFIT(filename=None, groupname=None, nref=5, niter_optical=500, beta_optical
print( nomi_rois)
print( O_spots_list)
for iterm, (O_spots, name, ROI ) in enumerate(zip(O_spots_list, nomi_rois, rois)):
if (iterm)%nprocs == myrank:
# print " MYRANK %d fa "%myrank, iterm
......@@ -1196,10 +1199,10 @@ def reload_trajectories(trajectory_file, nomi_rois) : ### , trajectory_file_grou
for iterm, name in enumerate( nomi_rois[:]):
h5group = h5[name]
trajectory = Trajectory()
trajectory.X.intercept = h5group["Xintercept"].value
trajectory.X.slope = h5group["Xslope"].value
trajectory.Y.intercept = h5group["Yintercept"].value
trajectory.Y.slope = h5group["Yslope"].value
trajectory.X.intercept = h5group["Xintercept"][()]
trajectory.X.slope = h5group["Xslope"][()]
trajectory.Y.intercept = h5group["Yintercept"][()]
trajectory.Y.slope = h5group["Yslope"][()]
# trajectory.N = O_spots.shape[0]
trajectory_list.append( trajectory )
return trajectory_list
......@@ -1209,17 +1212,22 @@ def reload_trajectories(trajectory_file, nomi_rois) : ### , trajectory_file_grou
def DOROIS(filename = "../nonregressions/demo_imaging.hdf5" , groupname = "ROI_B/foil_scanXX/scans/Scan273/",
roi_filename = None,
roisgroupname = "ROI_B/",
target_filename="newscan.h5",
roisgroupname_target = "ROI_B_FIT8/",
newscanstarget = "scanXX/scans/Scan273",
responsefilename = "responses.h5",
resynth_z_square = None,
responsepath = "fit",
nex = 3,
filter_rois=1,
recenterings= None,
filterMask = None):
if roi_filename is None:
roi_filename = filename
O_spots_list, nomi_rois, stats , origini, rois, xscales, energy= get_spots_list( filename, groupname , filter_rois= filter_rois )
trajectory_list = []
......@@ -1231,7 +1239,6 @@ def DOROIS(filename = "../nonregressions/demo_imaging.hdf5" , groupname = "ROI_B
for iterm, (O_spots, name) in enumerate(zip(O_spots_list[:], nomi_rois[:])):
if (iterm)%nprocs == myrank:
if myrank ==0 :
......@@ -1239,13 +1246,16 @@ def DOROIS(filename = "../nonregressions/demo_imaging.hdf5" , groupname = "ROI_B
h5group = h5[name]
print( name, "data")
solution = h5group["data"][:]
solution = h5group["data"][()]
trajectory = Trajectory()
trajectory.X.intercept = h5group["Xintercept"].value
trajectory.X.slope = h5group["Xslope"].value
trajectory.Y.intercept = h5group["Yintercept"].value
trajectory.Y.slope = h5group["Yslope"].value
nref = h5group["nref"].value
trajectory.X.intercept = h5group["Xintercept"][()]
trajectory.X.slope = h5group["Xslope"][()]
trajectory.Y.intercept = h5group["Yintercept"][()]
trajectory.Y.slope = h5group["Yslope"][()]
nref = h5group["nref"][()]
if resynth_z_square is not None:
solution = gaussian_filter( solution, [ nref*resynth_z_square , 0 ])
trajectory.N = O_spots.shape[0]
if recenterings is not None:
......@@ -1261,7 +1271,7 @@ def DOROIS(filename = "../nonregressions/demo_imaging.hdf5" , groupname = "ROI_B
trajectory.Y.intercept += -recy
if "line" in h5group:
trajectory.line = h5group["line"].value
trajectory.line = h5group["line"][()]
lh,lslope =trajectory.line
lh = lh-recy + recx*lslope
trajectory.line = np.array([lh,lslope])
......@@ -1304,13 +1314,15 @@ def DOROIS(filename = "../nonregressions/demo_imaging.hdf5" , groupname = "ROI_B
# print( filename)
# print( roisgroupname+ "/rois_definition/image")
h5f = h5py.File(filename,"r")
h5f = h5py.File( roi_filename,"r")
# imagealldect = h5f [roisgroupname+ "/rois_definition/image"][:]
print( roi_filename)
print( roisgroupname+ "/image" )
imagealldect = h5f [roisgroupname+ "/image"][:]
# h5 = h5f [roisgroupname+ ("/rois_definition/rois_dict/ROI%02d"%int(name)) ]
h5 = h5f [roisgroupname+ ("/ROI%02d"%int(name)) ]
h5 = h5f [roisgroupname+ ("/rois_dict/ROI%02d"%int(name)) ]
mask = h5["mask" ][:]
origin = h5["origin"][:]
h5f.close()
......@@ -1427,7 +1439,17 @@ def DOROIS(filename = "../nonregressions/demo_imaging.hdf5" , groupname = "ROI_B
h5.require_group( str(name) )
h5 = h5 [str(name) ]
for i in range(len(newspots)):
spot_sum = newspots[i].sum()
if spot_sum:
newspots[i] = newspots[i]/spot_sum
h5["matrix"] = newspots
h5["Xintercept"] =trajectory.X.intercept
h5["Yintercept"] =trajectory.Y.intercept
h5["Xslope"] =trajectory.X.slope
......
......@@ -267,6 +267,7 @@ class mainwindow(Qt.QMainWindow):
self.close()
def confirm_and_Exit(self):
self.isOK=1
self.roiobj_returned = self.getRoiObj()
self.close()
def remoteMaskload(self):
......@@ -1272,8 +1273,15 @@ def launch4MatPlotLib(layout=None, im4roi=None, manageQApp = True):
if manageQApp:
app.exec_()
if w4r.isOK:
return_obj = w4r.getRoiObj()
else:
return_obj = None
if dostop:
app.quit()
return return_obj
return w4r
......
......@@ -57,7 +57,7 @@ def my_denoise_tv_chambolle_positive(image, weight=0.1, n_iter_max=200):
def _denoise_tv_chambolle_nd(image, weight=0.1, eps=2.e-4, n_iter_max=200,
positivity=False):
positivity=False, cdim = None):
"""Perform total-variation denoising on n-dimensional images.
Parameters
......@@ -139,7 +139,10 @@ def _denoise_tv_chambolle_nd(image, weight=0.1, eps=2.e-4, n_iter_max=200,
slices_d[ax] = slice(1, None)
slices_p[ax+1] = slice(0, -1)
slices_p[0] = ax
if cdim is None:
d[tuple(slices_d)] += p[tuple(slices_p)]
else:
d[tuple(slices_d)] += p[tuple(slices_p)] *cdim[ax]
slices_d[ax] = slice(None)
slices_p[ax+1] = slice(None)
out_nopos = image + d
......@@ -161,7 +164,11 @@ def _denoise_tv_chambolle_nd(image, weight=0.1, eps=2.e-4, n_iter_max=200,
for ax in range(ndim):
slices_g[ax+1] = slice(0, -1)
slices_g[0] = ax