Skip to content
GitLab
Commit 00233a02 authored by myron's avatar myron
Browse files

restructuring the volume extraction and xrs_swissknife

parent 23322b8e
Expand all Hide whitespace changes
Inline Side-by-side
XRStools/XRS_swissknife.py
View file @ 00233a02
This diff is collapsed.
XRStools/xrs_read.py
View file @ 00233a02
...@@ -2164,7 +2164,7 @@ class read_id20: ...@@ -2164,7 +2164,7 @@ class read_id20:
self.scans = {} # was a dictionary before self.scans = {} # was a dictionary before
if absfilename is not None: if absfilename is not None:
if not os.path.isfile(absfilename): if not os.path.isfile(absfilename):
raise Exception('IOError! No such file, please check filename.') raise Exception('IOError! No such file %s, please check filename.' % absfilename )
self.path = os.path.split(absfilename)[0] + '/' self.path = os.path.split(absfilename)[0] + '/'
self.filename = os.path.split(absfilename)[1] self.filename = os.path.split(absfilename)[1]
if not edfName: if not edfName:
......
nonregressions/volumes/interpola_1/batch_extraction_esynth1.py
View file @ 00233a02
...@@ -3,46 +3,64 @@ import h5py ...@@ -3,46 +3,64 @@ import h5py
import glob import glob
import json import json
import os import os
import h5py
BATCH_PARALLELISM = 4 BATCH_PARALLELISM = 4
import os import os
def main(): def main():
peaks_shifts = File("peaks_positions_for_analysers.h5","w")["peaks_positions"] peaks_shifts = h5py.File("peaks_positions_for_analysers.h5","r")["peaks_positions"][()]
assert( len(peaks_shifts) == 72)) assert( len(peaks_shifts) == 72)
Enominal = np.median(peak_shifts) os.system("xz -dk mask.h5.xz")
peak_shifts-= Enominal Enominal = np.median(peaks_shifts)
peaks_shifts-= Enominal
datadir = "/data/id20/inhouse/data/run3_20/run3_es949" datadir = "/data/id20/inhouse/data/run3_20/run3_es949"
os.system("mkdir results") os.system("mkdir results")
roi_target_path = "results/myrois.h5:/ROIS" roi_target_path = "results/myrois.h5:/ROIS"
filter_path : "mask.h5:/FILTER_MASK/filter" # roi_target_path = "rr.h5:/extracted/ROI_AS_SELECTED"
roi_scan_num = 246 filter_path = "mask.h5:/FILTER_MASK/filter"
roi_scan_num = [245,246,247]
first_scan_num = 651 first_scan_num = 651
Ydim = 25 Ydim = 25
Zdim = 10 Zdim = 10
Edim = 7 Edim = 7
monitorcolumn = "izero/0.000001" monitor_column = "izero/0.000001"
signals_target_file = "results/signals.h5" signals_target_file = "results/signals.h5"
interpolated_signals_target_file = "results/interpolated_signals.h5" interpolated_signals_target_file = "results/interpolated_signals.h5"
extracted_reference_target_file = "results/reference.h5"
reference_scan_list = [245, 246, 247]
# If reference_clip is not None, then a smaller part of the reference scan is considered
# This may be usefule to obtain smaller volumes containing the interesting part
# The used reference scan will the correspond to the positions from reference_clip[0] to reference_clip[1]-1 included
###########
# reference_clip = None
reference_clip = [ 90, 180 ]
## in the reference scan for each position there is a spot with a maximum. We set zero the background which is further than
## such radius from the maximum
isolate_spot_by = 6
steps_to_do = { steps_to_do = {
"do_step_make_roi": False, "do_step_make_roi": False,
"do_step_interpolation": True,
"do_step_sample_extraction": False, "do_step_sample_extraction": False,
"do_step_interpolation": False,
"do_extract_reference_scan": True,
"do_step_make_reference": False, "do_step_make_reference": False,
"do_step_scalar_products": False, "do_step_scalar_products": False,
"do_step_finalise_for_fit": True "do_step_finalise_for_fit": False
} }
...@@ -60,8 +78,13 @@ def main(): ...@@ -60,8 +78,13 @@ def main():
Edim = Edim , Edim = Edim ,
monitor_column = monitor_column, monitor_column = monitor_column,
signals_target_file = signals_target_file signals_target_file = signals_target_file,
interpolated_signals_target_file = interpolated_signals_target_file interpolated_signals_target_file = interpolated_signals_target_file,
reference_scan_list = reference_scan_list,
reference_clip = reference_clip,
extracted_reference_target_file = extracted_reference_target_file ,
isolate_spot_by = isolate_spot_by
) )
...@@ -87,29 +110,59 @@ def process_input(s, go=0, exploit_slurm_mpi = 0, stop_omp = False): ...@@ -87,29 +110,59 @@ def process_input(s, go=0, exploit_slurm_mpi = 0, stop_omp = False):
def select_rois( datadir = None, roi_scan_num=None, roi_target_path = None, filter_path = None): def select_rois( datadir = None, roi_scan_num=None, roi_target_path = None, filter_path = None):
if np.isscalar(roi_scan_num):
scans = [roi_scan_num]
else:
scans = list(roi_scan_num)
inputstring = """ inputstring = """
create_rois: create_rois:
expdata : {expdata} expdata : {expdata}
scans : [{roi_scan_num}] scans : {scans}
roiaddress : {roi_target_path} roiaddress : {roi_target_path}
filter_path : {filter_path} filter_path : {filter_path}
""".format( """.format(
expdata = os.path.join( datadir, "/hydra"), expdata = os.path.join( datadir, "hydra"),
roi_scan_num = roi_scan_num, scans = scans,
roi_target_path = roi_target_path, roi_target_path = roi_target_path,
filter_path = filter_path filter_path = filter_path
) )
process_input( inputstring, exploit_slurm_mpi = 0 ) process_input( inputstring, exploit_slurm_mpi = 0 )
extract_sample_givenrois( def get_reference( roi_path = None,
roi_path = roi_path, datadir = None,
Start = first_scan_num , reference_scan_num=None,
End = (first_scan_num + Zdim * Edim ) , monitor_column = None,
Thickness = Zdim extracted_reference_target_file = None,
) isolate_spot_by = None,
):
signal_path = extracted_reference_target_file + ":/"
inputstring = """
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitor_column : {monitor_column}
scan_interval : [{reference_scan_num},{reference_scan_num_plus1} ]
signaladdress : {signal_path}
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s=inputstring.format(
exp_data = os.path.join( datadir, "hydra"),
reference_scan_num = reference_scan_num,
reference_scan_num_plus1 = reference_scan_num+1,
monitor_column = monitor_column,
roi_path = roi_path,
isolate_spot_by = isolate_spot_by,
signal_path = signal_path
)
process_input( s , exploit_slurm_mpi = 1)
def extract_sample_givenrois( def extract_sample_givenrois(
...@@ -118,8 +171,8 @@ def extract_sample_givenrois( ...@@ -118,8 +171,8 @@ def extract_sample_givenrois(
Start = None, Start = None,
End = None, End = None,
Thickness = None , Thickness = None ,
monitor_columns = None, monitor_column = None,
signals_target_file = None signals_target_file = None,
): ):
for start in range(Start,End, Thickness): for start in range(Start,End, Thickness):
...@@ -137,116 +190,112 @@ def extract_sample_givenrois( ...@@ -137,116 +190,112 @@ def extract_sample_givenrois(
monitor_column : {monitor_column} monitor_column : {monitor_column}
sumto1D : 0 sumto1D : 0
""".format( """.format(
expdata = os.path.join( datadir, "/hydra"), expdata = os.path.join( datadir, "hydra"),
roi_path = roi_path, roi_path = roi_path,
start = start, start = start,
end = end, end = end,
monitor_column = monitor_column, monitor_column = monitor_column,
signal_path = signal_path signal_path = signal_path
) )
process_input(s, exploit_slurm_mpi = 1) process_input(inputstring, exploit_slurm_mpi = 1)
class interpolate( peaks_shifts, interp_file, interp_file_target): def interpolate( peaks_shifts, interp_file_str, interp_file_target_str):
volum_list = list(interp_file.keys())
scan_num_list = np.array([ int( t.split("_") [1]) for t in volum_list])
ene_list = np.array([ interp_file[vn]["scans"]["Scan%03d"%sn ]["motorDict"]["energy"].value for vn,sn in zip(volum_list, scan_num_list ) ])
interp_file = h5py.File( interp_file_str ,"r+")
print ( " ecco la scannumlist " , scan_num_list) interp_file_target = h5py.File( interp_file_target_str ,"r+")
print (" ecco ene_list", ene_list)
order = np.argsort( ene_list )
ene_list = ene_list [order]
scan_num_list = scan_num_list [order]
volum_list = [ volum_list [ii] for ii in order ]
print ( " ECCO I DATI ")
print ( ene_list )
print ( cenom )
# raise
for t_vn, t_sn, t_ene in list(zip(volum_list, scan_num_list, ene_list ))[0:]:
rois_coeffs={}
for roi_num, de in enumerate( cenom ):
print ( roi_num, "===== " , t_ene+de , ene_list .min() , t_ene+de , ene_list .max() )
if t_ene+de < ene_list .min() or t_ene+de > ene_list .max():
continue
print ( " CONTINUO ", t_ene+de, ene_list .min() ,ene_list .max() )
i0 = np.searchsorted( ene_list , t_ene+de )-1
assert(i0>=0)
i1=i0+1
print (i0, i1, len(ene_list))
print (ene_list)
assert(i1<len( ene_list ))
DE = ( ene_list[i1] - ene_list[i0] )
df = ( t_ene+de - ene_list[i0] )/ DE
rois_coeffs[ roi_num ] = [ i0,(1-df) , i1,df ]
print ( " for reinterpolation of ", t_vn ," interpolation scheme is the following ", rois_coeffs)
volum_list = list(interp_file.keys())
interp_file_target.require_group(target) scan_num_list = np.array([ int( t.split("_") [1]) for t in volum_list])
ene_list = np.array([ interp_file[vn]["scans"]["Scan%03d"%sn ]["motorDict"]["energy"].value for vn,sn in zip(volum_list, scan_num_list ) ])
fscans = interp_file[ t_vn ]["scans"]
keys_list = list( fscans.keys() )
print ( " keylist ", keys_list)
# interp_file_target.flush() print ( " ecco la scannumlist " , scan_num_list)
print (" ecco ene_list", ene_list)
fscans = interp_file_target[ target+"/"+t_vn ]["scans"]
keys_list = list( fscans.keys() ) order = np.argsort( ene_list )
print ( " keylist ", keys_list)
ene_list = ene_list [order]
scan_num_list = scan_num_list [order]
volum_list = [ volum_list [ii] for ii in order ]
# raise
for t_vn, t_sn, t_ene in list(zip(volum_list, scan_num_list, ene_list ))[0:]:
rois_coeffs={}
for roi_num, de in enumerate( peaks_shifts ):
print ( roi_num, "===== " , t_ene+de , ene_list .min() , t_ene+de , ene_list .max() )
if t_ene+de < ene_list .min() or t_ene+de > ene_list .max():
continue
print ( " CONTINUO ", t_ene+de, ene_list .min() ,ene_list .max() )
i0 = np.searchsorted( ene_list , t_ene+de )-1
assert(i0>=0)
i1=i0+1
print (i0, i1, len(ene_list))
print (ene_list)
assert(i1<len( ene_list ))
DE = ( ene_list[i1] - ene_list[i0] )
df = ( t_ene+de - ene_list[i0] )/ DE
rois_coeffs[ roi_num ] = [ i0,(1-df) , i1,df ]
print ( " for reinterpolation of ", t_vn ," interpolation scheme is the following ", rois_coeffs)
print ( " roislist keylist", list(rois_coeffs.keys()) )
fscans = interp_file[ t_vn ]["scans"]
keys_list = list( fscans.keys() )
print ( " keylist ", keys_list)
# interp_file_target.flush()
fscans = interp_file_target[ target+"/"+t_vn ]["scans"]
keys_list = list( fscans.keys() )
print ( " keylist ", keys_list)
print ( " roislist keylist", list(rois_coeffs.keys()) )
for k in keys_list:
if k[:3]=="ROI":
if int(k[3:]) not in rois_coeffs:
print (" rimuovo ", k)
del fscans[k]
for sn in range(t_sn, t_sn+1):
fScan = fscans["Scan%03d"% sn]
keys_list = list( fScan.keys() )
for k in keys_list: for k in keys_list:
if k[:3]=="ROI": if k!="motorDict":
if int(k[3:]) not in rois_coeffs: if int(k) not in rois_coeffs:
print (" rimuovo ", k) print (" rimuovo da scans", k)
del fscans[k] del fScan[k]
for sn in range(t_sn, t_sn+1):
fScan = fscans["Scan%03d"% sn]
keys_list = list( fScan.keys() ) for sn in range(t_sn, t_sn+1):
for k in keys_list: fScan = fscans["Scan%03d"% sn]
if k!="motorDict": keys_list = list( fScan.keys() )
if int(k) not in rois_coeffs: for k in keys_list:
print (" rimuovo da scans", k) if k!="motorDict":
del fScan[k] assert( int(k) in rois_coeffs)
k = int(k)
i0,f0,i1,f1 = rois_coeffs[k]
for sn in range(t_sn, t_sn+1):
fScan = fscans["Scan%03d"% sn] matrix0 = interp_file[volum_list[i0] ]["scans"]["Scan%03d"%( scan_num_list[i0]+sn-t_sn) ][str(k)]["matrix"][:]
keys_list = list( fScan.keys() ) matrix1 = interp_file[volum_list[i1] ]["scans"]["Scan%03d"%( yyyscan_num_list[i1]+sn-t_sn) ][str(k)]["matrix"][:]
for k in keys_list: monitor = np.ones( matrix0.shape[0],"f" )
if k!="motorDict": newmatrix = f0* matrix0+f1*matrix1
assert( int(k) in rois_coeffs)
k = int(k) if "matrix" in fScan[str(k)] :
i0,f0,i1,f1 = rois_coeffs[k] del fScan[str(k)]["matrix"]
if "monitor" in fScan[str(k)] :
matrix0 = interp_file[volum_list[i0] ]["scans"]["Scan%03d"%( scan_num_list[i0]+sn-t_sn) ][str(k)]["matrix"][:] del fScan[str(k)]["monitor"]
matrix1 = interp_file[volum_list[i1] ]["scans"]["Scan%03d"%( yyyscan_num_list[i1]+sn-t_sn) ][str(k)]["matrix"][:] if "monitor_divider" in fScan[str(k)] :
monitor = np.ones( matrix0.shape[0],"f" ) del fScan[str(k)]["monitor_divider"]
newmatrix = f0* matrix0+f1*matrix1
fScan[str(k)]["matrix"] = newmatrix
if "matrix" in fScan[str(k)] : fScan[str(k)]["monitor"] = monitor
del fScan[str(k)]["matrix"] fScan[str(k)]["monitor_divider"] = 1.0
if "monitor" in fScan[str(k)] :
del fScan[str(k)]["monitor"]
if "monitor_divider" in fScan[str(k)] :
del fScan[str(k)]["monitor_divider"]
fScan[str(k)]["matrix"] = newmatrix
fScan[str(k)]["monitor"] = monitor
fScan[str(k)]["monitor_divider"] = 1.0
...@@ -270,8 +319,14 @@ def tools_sequencer( peaks_shifts = None, ...@@ -270,8 +319,14 @@ def tools_sequencer( peaks_shifts = None,
##################################################### #####################################################
## can be left to None, will be set to the used target ## can be left to None, will be set to the used target
roi_path = None, roi_path = None,
signals_file = None signals_file = None,
interpolated_signals_file = None interpolated_signals_file = None,
reference_clip = None,
isolate_spot_by = None,
reference_scan_list = None,
extracted_reference_target_file = None ,
) : ) :
...@@ -286,8 +341,7 @@ def tools_sequencer( peaks_shifts = None, ...@@ -286,8 +341,7 @@ def tools_sequencer( peaks_shifts = None,
select_rois(datadir = datadir , select_rois(datadir = datadir ,
roi_scan_num = roi_scan_num , roi_scan_num = roi_scan_num ,
roi_target_path = roi_target_path, roi_target_path = roi_target_path,
filter_path = filter_path, filter_path = filter_path
signals_target_file = signals_target_file
) )
if(steps_to_do["do_step_sample_extraction"]): if(steps_to_do["do_step_sample_extraction"]):
...@@ -296,14 +350,62 @@ def tools_sequencer( peaks_shifts = None, ...@@ -296,14 +350,62 @@ def tools_sequencer( peaks_shifts = None,
datadir = datadir, datadir = datadir,
Start = first_scan_num , Start = first_scan_num ,
End = (first_scan_num + Zdim * Edim ) , End = (first_scan_num + Zdim * Edim ) ,
Thickness = Zdim Thickness = Zdim,
monitor_column = monitor_column, monitor_column = monitor_column,
signals_target_file = signals_target_file signals_target_file = signals_target_file
) )
if(steps_to_do["do_step_interpolation"]): if(steps_to_do["do_step_interpolation"]):
os.system("cp {signals_file} {interpolated_signals_target_file}".format(signals_file=signals_file, interpolated_signals_target_file =interpolated_signals_target_file) ) os.system("cp {signals_file} {interpolated_signals_target_file}".format(signals_file=signals_file, interpolated_signals_target_file =interpolated_signals_target_file) )
Interpolate( peaks_shift , signals_file , interpolated_signals_target_file ) interpolate( peaks_shifts , signals_file , interpolated_signals_target_file )
if(steps_to_do["do_extract_reference_scan"]): # of course we need the REFERENCE SCAN
for reference_scan_num in reference_scan_list:
get_reference( datadir = datadir,
monitor_column = monitor_column,
extracted_reference_target_file = extracted_reference_target_file,
isolate_spot_by = isolate_spot_by,
reference_scan_num = reference_scan_num
)
# for other in other_rois_for_ref:
# os.system("cp roi_%d.h5 roi_%d.h5"%(roi_scann, other) )
# if reference_clip is not None:
# clip1, clip2= reference_clip
# ftarget = h5py.File( "roi_%d.h5" % roi_scann ,"r+")
# target_group = ftarget["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% roi_scann ]
# for k in target_group.keys():
# if k != "motorDict":
# print(" SHRINKING scan for ROI %s in file roi_%d.h5 " %( k, roi_scann ))
# for dsn in ["matrix", "monitor", "xscale"]:
# mat = target_group[k][dsn][()]
# del target_group[k][dsn]
# target_group[k][dsn] = mat[clip1:clip2]
# ftarget.close()
# for other in other_rois_for_ref:
# get_reference(roi_scan_num=other)
# ftarget = h5py.File( "roi_%d.h5" % roi_scann ,"r+")
# fsource = h5py.File( "roi_%d.h5" % other , "r")
# source_group = fsource["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% other ]
# target_group = ftarget["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% roi_scann ]
# for k in target_group.keys():
# if k != "motorDict":
# print(" ADDING data for ROI %s from file roi_%d.h5 " %( k, other ))
# mat = source_group[k]["matrix"][()]
# if clip1 is not None:
# mat = mat[clip1:clip2]
# target_group[k]["matrix"][:] += mat
# print( " SUCCESS ")
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment