Skip to content
GitLab
Commit 921ddb77 authored by myron's avatar myron
Browse files

the sequencer is now a module

parent fbbc1685
Expand all Hide whitespace changes
Inline Side-by-side
XRStools/tools_sequencer_esynth.py 0 → 100644
View file @ 921ddb77
import numpy as np
import h5py
import glob
import json
import os
import h5py
import math
BATCH_PARALLELISM = 1
import os
def process_input(s, go=0, exploit_slurm_mpi = 0, stop_omp = False):
open("input_tmp_%d.par"%go, "w").write(s)
background_activator = ""
if (go % BATCH_PARALLELISM ):
background_activator = "&"
prefix=""
if stop_omp:
prefix = prefix +"export OMP_NUM_THREADS=1 ;"
if ( exploit_slurm_mpi==0 ):
os.system(prefix +"mpirun -n 1 XRS_swissknife input_tmp_%d.par %s"%(go, background_activator))
elif ( exploit_slurm_mpi>0 ):
os.system(prefix + "mpirun XRS_swissknife input_tmp_%d.par %s"%(go, background_activator) )
else:
os.system(prefix + "mpirun -n %d XRS_swissknife input_tmp_%d.par %s"%(abs( exploit_slurm_mpi ), go, background_activator) )
def select_rois( datadir = None, roi_scan_num=None, roi_target_path = None, filter_path = None):
if np.isscalar(roi_scan_num):
scans = [roi_scan_num]
else:
scans = list(roi_scan_num)
input_string = """
create_rois:
expdata : {expdata}
scans : {scans}
roiaddress : {roi_target_path}
filter_path : {filter_path}
""".format(
expdata = os.path.join( datadir, "hydra"),
scans = scans,
roi_target_path = roi_target_path,
filter_path = filter_path
)
process_input( input_string, exploit_slurm_mpi = 0 )
def get_reference( roi_path = None,
datadir = None,
reference_scan_list = None,
monitor_column = None,
extracted_reference_target_file = None,
isolate_spot_by = None
):
signal_path = extracted_reference_target_file + ":/"
input_string = """
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitorcolumn : {monitor_column}
scan_list : {reference_scan_list}
signaladdress : "{extracted_reference_target_file}:/references"
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s=input_string.format(
expdata = os.path.join( datadir, "hydra"),
reference_scan_list = reference_scan_list,
monitor_column = monitor_column,
roi_path = roi_path,
isolate_spot_by = isolate_spot_by,
signal_path = signal_path,
extracted_reference_target_file = extracted_reference_target_file
)
process_input( s , exploit_slurm_mpi = 0)
def extract_sample_givenrois(
roi_path = None,
datadir = None,
Start = None,
End = None,
Thickness = None ,
monitor_column = None,
signals_target_file = None,
):
for iE, start in enumerate(range(Start,End, Thickness)):
end = start+Thickness
signal_path = signals_target_file + ":/E{iE}".format(iE=iE)
input_string = """
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
scan_interval : [{start}, {end}]
energycolumn : sty
signaladdress : {signal_path}
monitorcolumn : {monitor_column}
sumto1D : 0
""".format(
expdata = os.path.join( datadir, "hydra"),
roi_path = roi_path,
start = start,
end = end,
monitor_column = monitor_column,
signal_path = signal_path
)
process_input(input_string, exploit_slurm_mpi = 0)
def resynthetise_scan(
old_scan_address= None,
response_file = None ,
target_address = None,
original_roi_path = None
):
input_string = """
superR_recreate_rois :
### we have calculated the responses in responsefilename
### and we want to enlarge the scan by a margin of 3 times
### the original scan on the right and on the left
### ( so for a total of a 7 expansion factor )
responsefilename : {response_file}
nex : 0
## the old scan covered by the old rois
old_scan_address : {old_scan_address}
## where new rois and bnew scan are written
target_filename : {target_address}
filter_rois : 0
original_roi_path : {original_roi_path}
"""
s=input_string.format( response_file = response_file , target_address = target_address, old_scan_address=old_scan_address, original_roi_path = original_roi_path +"/rois_definition")
process_input( s , exploit_slurm_mpi = 0, stop_omp = True)
def get_scalars( iE = None,
signals_file = None,
reference_file = None,
target_file = None,
selected_rois = None
):
inputstring = """
superR_scal_deltaXimages_Esynt :
sample_address : {signals_file}:/E{iE}/scans
delta_address : {reference_file}:/rois_and_reference/scans/ScansSum
load_factors_from :
nbin : 1
target_address : {target_file}:/E{iE}/scal_prods
roi_keys : {selected_rois}
""" . format( iE = iE,
signals_file = signals_file ,
reference_file = reference_file ,
target_file = target_file,
selected_rois = list( selected_rois )
)
process_input( inputstring, exploit_slurm_mpi = 0)
def InterpInfo_Esynt_components(peaks_shifts , signals_file=None, custom_ene_list = None, custom_components = None, selected_rois = None ):
f = h5py.File(signals_file,"r")
volumes_list = list(f.keys())
ene_list = []
for v in volumes_list:
dg_Scans = [ t for t in f[v]["scans"].keys() if t[:4]=="Scan"]
ene_list.append( f[v]["scans"][dg_Scans[0]] ["motorDict"]["energy"][()] )
components = h5py.File( custom_components ,"r")["components"] [()]
info_dict = {}
for i_interval in range(len(components)):
info_dict[str(i_interval)] = {}
info_dict[str(i_interval)]["E"] = custom_ene_list[ i_interval ]
info_dict[str(i_interval)]["coefficients"]={}
for i_n in range(len(energy_exp_grid)):
info_dict[str(i_interval)]["coefficients" ][ str(i_n) ]={}
for roi_num, de in enumerate( peaks_shifts ):
info_dict[str(i_interval)]["coefficients" ][ str(i_n) ][ str(roi_num) ] = 0
for ic in range(len(components)):
for i_interval in range(len(custom_ene_list)-1):
cE1 = custom_ene_list[ i_interval ]
cE2 = custom_ene_list[ i_interval+1 ]
for i_ene, t_ene in enumerate( energy_exp_grid) :
for roi_num, de in enumerate( peaks_shifts ):
if selected_rois and roi_num not in selected_rois:
continue
if t_ene+de < cE1 or t_ene+de > cE2:
continue
alpha = (cE2-(t_ene+de) )/(cE2+cE1)
info_dict[str(ic)]["coefficients" ][ str(i_ene) ][ str(roi_num) ] += alpha * components[ic][ i_interval ]
info_dict[str(ic)]["coefficients" ][ str(i_ene) ][ str(roi_num) ] += (1-alpha)*components[ic][ i_interval+1 ]
return info_dict
def InterpInfo_Esynt( peaks_shifts ,
signals_file = None,
custom_ene_list = None,
selected_rois = None
):
f = h5py.File(signals_file,"r")
volumes_list = list(f.keys())
ene_list = []
for v in volumes_list:
dg_Scans = [ t for t in f[v]["scans"].keys() if t[:4]=="Scan"]
ene_list.append( f[v]["scans"][dg_Scans[0]] ["motorDict"]["energy"][()] )
print ( " Here custom ene list " , custom_ene_list)
print (" Here all the analyser energies ", np.sort(np.add.outer( ene_list , peaks_shifts[selected_rois]).ravel()) )
order = list(np.argsort( ene_list ))
ene_list = [ ene_list[ii] for ii in order]
volumes_list = [ volumes_list[ii] for ii in order]
info_dict = {}
for i_intervallo in range(len(custom_ene_list)):
info_dict[str(i_intervallo)] = {}
info_dict[str(i_intervallo)]["E"] = custom_ene_list[ i_intervallo ]
info_dict[str(i_intervallo)]["coefficients"]={}
for t_vn, t_ene in list(zip( volumes_list, ene_list )):
info_dict[str(i_intervallo)]["coefficients" ][ t_vn ]={}
for i_intervallo in range(len( custom_ene_list)-1):
cE1 = custom_ene_list[ i_intervallo ]
cE2 = custom_ene_list[ i_intervallo+1 ]
for t_vn, t_ene in list(zip(volumes_list, ene_list ))[0:]:
for roi_num, de in enumerate( peaks_shifts ):
if selected_rois and roi_num not in selected_rois:
continue
if t_ene+de < cE1 or t_ene+de > cE2:
continue
alpha = (cE2-(t_ene+de) )/(cE2-cE1)
info_dict[str(i_intervallo)]["coefficients" ][ str(t_vn) ][ str(roi_num) ] = alpha
info_dict[str(i_intervallo+1)]["coefficients"][ str(t_vn) ][ str(roi_num) ] = 1-alpha
return info_dict
def tools_sequencer( peaks_shifts = None,
datadir = None,
filter_path = None,
roi_scan_num = None,
roi_target_path = None,
first_scan_num = None ,
Ydim = None ,
Zdim = None ,
Edim = None ,
monitor_column = None,
signals_target_file = None,
steps_to_do = None,
#####################################################
## can be left to None, will be set to the used target
roi_path = None,
signals_file = None,
reference_clip = None,
isolate_spot_by = None,
reference_scan_list = None,
extracted_reference_target_file = None ,
response_target_file = None,
response_fit_options = None,
resynthetised_reference_and_roi_target_file = None,
selected_rois = None,
scalar_products_target_file = None,
volume_retrieval_beta = None,
volume_retrieval_niter = None ,
energy_custom_grid = None,
custom_components_file = None ,
interpolation_infos_file = None
) :
if roi_path is None:
roi_path = roi_target_path
if signals_file is None:
signals_file = signals_target_file
if(steps_to_do["do_step_make_roi"]): # ROI selection and reference scan
select_rois(datadir = datadir ,
roi_scan_num = roi_scan_num ,
roi_target_path = roi_target_path,
filter_path = filter_path
)
if(steps_to_do["do_step_sample_extraction"]):
extract_sample_givenrois(
roi_path = roi_path,
datadir = datadir,
Start = first_scan_num ,
End = (first_scan_num + Zdim * Edim ) ,
Thickness = Zdim,
monitor_column = monitor_column,
signals_target_file = signals_target_file
)
if(steps_to_do["do_step_extract_reference_scan"]): # of course we need the REFERENCE SCAN
get_reference( datadir = datadir,
roi_path = roi_path,
monitor_column = monitor_column,
extracted_reference_target_file = extracted_reference_target_file,
isolate_spot_by = isolate_spot_by,
reference_scan_list = reference_scan_list
)
if reference_clip is not None:
clip1, clip2= reference_clip
ftarget = h5py.File( extracted_reference_target_file ,"r+")
for roi_scann in reference_scan_list:
target_group = ftarget["references/scans/Scan%03d"% roi_scann ]
for k in target_group.keys():
if k != "motorDict":
print(" SHRINKING scan for ROI %s in file roi_%d.h5 " %( k, roi_scann ))
for dsn in ["matrix", "monitor", "xscale"]:
mat = target_group[k][dsn][()]
del target_group[k][dsn]
target_group[k][dsn] = mat[clip1:clip2]
ftarget.close()
ftarget = h5py.File( extracted_reference_target_file ,"r+")
ftarget["references/scans/ScansSum"] = ftarget["references/scans/Scan%03d"% reference_scan_list[0] ]
for other in reference_scan_list[1:]:
source_group = ftarget["references/scans/Scan%03d"% other ]
target_group = ftarget["references/scans/ScansSum" ]
for k in target_group.keys():
if k != "motorDict":
print(" ADDING data for ROI %s from file roi_%d.h5 " %( k, other ))
mat = source_group[k]["matrix"][()]
target_group[k]["matrix"][:] += mat
ftarget.close()
if(steps_to_do["do_step_fit_reference_response"]):
synthetise_response(
scan_address= extracted_reference_target_file +":references/scans/ScansSum" ,
target_address = response_target_file +":/FIT",
response_fit_options = response_fit_options
)
if(steps_to_do["do_step_resynthetise_reference"]):
resynthetise_scan(
old_scan_address= extracted_reference_target_file +":references/scans/ScansSum" ,
response_file = response_target_file +":/FIT",
target_address = resynthetised_reference_and_roi_target_file + ":/rois_and_reference",
original_roi_path = roi_path
)
if(steps_to_do["do_step_scalars"]):
os.system("rm %s"%scalar_products_target_file)
for iE in range(Edim) :
get_scalars( iE = iE,
signals_file = signals_file,
reference_file = resynthetised_reference_and_roi_target_file ,
target_file = scalar_products_target_file,
selected_rois = selected_rois
)
if(steps_to_do["do_step_interpolation_coefficients"]):
if custom_components_file is None:
info_dict = InterpInfo_Esynt( peaks_shifts ,
signals_file = signals_file,
custom_ene_list = energy_custom_grid,
selected_rois = selected_rois
)
else:
info_dict = InterpInfo_Esynt_components( peaks_shifts,
signals_file = signals_file,
custom_ene_list = energy_custom_grid,
custom_components = custom_components_file,
selected_rois = selected_rois
)
json.dump(info_dict,open( interpolation_infos_file,"w"), indent=4)
if( steps_to_do["do_step_finalise_for_fit"] ):
get_volume_Esynt(
scalar_products_file = scalar_products_target_file,
interpolation_infos_file = interpolation_infos_file
)
def get_volume_Esynt(
scalar_products_file = None,
interpolation_infos_file = None
):
inputstring = """
superR_getVolume_Esynt :
scalprods_address : {scalar_products_file}:/
dict_interp : {interpolation_infos_file}
output_prefix : DATASFORCC/test0_
""".format(
scalar_products_file = scalar_products_file,
interpolation_infos_file = interpolation_infos_file
)
os.system("mkdir DATASFORCC")
process_input( inputstring, exploit_slurm_mpi = 0)
XRStools/tools_sequencer_interp.py 0 → 100644
View file @ 921ddb77
This diff is collapsed.
nonregressions/volumes/esrf_scans/esynth/batch_extraction_esynth1.py
View file @ 921ddb77
......@@ -6,8 +6,7 @@ import os
import h5py
import math
BATCH_PARALLELISM = 1
from XRStools import tools_sequencer_esynth
import os
def main():
......@@ -97,7 +96,7 @@ def main():
interpolation_infos_file = "interpolation_infos.json"
tools_sequencer( peaks_shifts = peaks_shifts ,
tools_sequencer_esynth.tools_sequencer( peaks_shifts = peaks_shifts ,
datadir = datadir ,
filter_path = filter_path ,
roi_scan_num = roi_scan_num ,
......@@ -137,457 +136,6 @@ def main():
)
def process_input(s, go=0, exploit_slurm_mpi = 0, stop_omp = False):
open("input_tmp_%d.par"%go, "w").write(s)
background_activator = ""
if (go % BATCH_PARALLELISM ):
background_activator = "&"
prefix=""
if stop_omp:
prefix = prefix +"export OMP_NUM_THREADS=1 ;"
if ( exploit_slurm_mpi==0 ):
os.system(prefix +"mpirun -n 1 XRS_swissknife input_tmp_%d.par %s"%(go, background_activator))
elif ( exploit_slurm_mpi>0 ):
os.system(prefix + "mpirun XRS_swissknife input_tmp_%d.par %s"%(go, background_activator) )
else:
os.system(prefix + "mpirun -n %d XRS_swissknife input_tmp_%d.par %s"%(abs( exploit_slurm_mpi ), go, background_activator) )
def select_rois( datadir = None, roi_scan_num=None, roi_target_path = None, filter_path = None):
if np.isscalar(roi_scan_num):
scans = [roi_scan_num]
else:
scans = list(roi_scan_num)
input_string = """
create_rois:
expdata : {expdata}
scans : {scans}
roiaddress : {roi_target_path}
filter_path : {filter_path}
""".format(
expdata = os.path.join( datadir, "hydra"),
scans = scans,
roi_target_path = roi_target_path,
filter_path = filter_path
)
process_input( input_string, exploit_slurm_mpi = 0 )
def get_reference( roi_path = None,
datadir = None,
reference_scan_list = None,
monitor_column = None,
extracted_reference_target_file = None,
isolate_spot_by = None
):
signal_path = extracted_reference_target_file + ":/"
input_string = """
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitorcolumn : {monitor_column}
scan_list : {reference_scan_list}
signaladdress : "{extracted_reference_target_file}:/references"
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s=input_string.format(
expdata = os.path.join( datadir, "hydra"),
reference_scan_list = reference_scan_list,
monitor_column = monitor_column,
roi_path = roi_path,
isolate_spot_by = isolate_spot_by,
signal_path = signal_path,
extracted_reference_target_file = extracted_reference_target_file
)
process_input( s , exploit_slurm_mpi = 0)
def extract_sample_givenrois(
roi_path = None,
datadir = None,
Start = None,
End = None,
Thickness = None ,
monitor_column = None,
signals_target_file = None,
):
for iE, start in enumerate(range(Start,End, Thickness)):
end = start+Thickness
signal_path = signals_target_file + ":/E{iE}".format(iE=iE)
input_string = """
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
scan_interval : [{start}, {end}]
energycolumn : sty
signaladdress : {signal_path}
monitorcolumn : {monitor_column}
sumto1D : 0
""".format(
expdata = os.path.join( datadir, "hydra"),
roi_path = roi_path,
start = start,
end = end,
monitor_column = monitor_column,
signal_path = signal_path
)
process_input(input_string, exploit_slurm_mpi = 0)
def resynthetise_scan(
old_scan_address= None,
response_file = None ,
target_address = None,
original_roi_path = None
):