Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ixstools
xrstools
Commits
921ddb77
Commit
921ddb77
authored
Dec 02, 2020
by
myron
Browse files
the sequencer is now a module
parent
fbbc1685
Changes
4
Expand all
Hide whitespace changes
Inline
Side-by-side
XRStools/tools_sequencer_esynth.py
0 → 100644
View file @
921ddb77
import
numpy
as
np
import
h5py
import
glob
import
json
import
os
import
h5py
import
math
BATCH_PARALLELISM
=
1
import
os
def
process_input
(
s
,
go
=
0
,
exploit_slurm_mpi
=
0
,
stop_omp
=
False
):
open
(
"input_tmp_%d.par"
%
go
,
"w"
).
write
(
s
)
background_activator
=
""
if
(
go
%
BATCH_PARALLELISM
):
background_activator
=
"&"
prefix
=
""
if
stop_omp
:
prefix
=
prefix
+
"export OMP_NUM_THREADS=1 ;"
if
(
exploit_slurm_mpi
==
0
):
os
.
system
(
prefix
+
"mpirun -n 1 XRS_swissknife input_tmp_%d.par %s"
%
(
go
,
background_activator
))
elif
(
exploit_slurm_mpi
>
0
):
os
.
system
(
prefix
+
"mpirun XRS_swissknife input_tmp_%d.par %s"
%
(
go
,
background_activator
)
)
else
:
os
.
system
(
prefix
+
"mpirun -n %d XRS_swissknife input_tmp_%d.par %s"
%
(
abs
(
exploit_slurm_mpi
),
go
,
background_activator
)
)
def
select_rois
(
datadir
=
None
,
roi_scan_num
=
None
,
roi_target_path
=
None
,
filter_path
=
None
):
if
np
.
isscalar
(
roi_scan_num
):
scans
=
[
roi_scan_num
]
else
:
scans
=
list
(
roi_scan_num
)
input_string
=
"""
create_rois:
expdata : {expdata}
scans : {scans}
roiaddress : {roi_target_path}
filter_path : {filter_path}
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
scans
=
scans
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
)
process_input
(
input_string
,
exploit_slurm_mpi
=
0
)
def
get_reference
(
roi_path
=
None
,
datadir
=
None
,
reference_scan_list
=
None
,
monitor_column
=
None
,
extracted_reference_target_file
=
None
,
isolate_spot_by
=
None
):
signal_path
=
extracted_reference_target_file
+
":/"
input_string
=
"""
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitorcolumn : {monitor_column}
scan_list : {reference_scan_list}
signaladdress : "{extracted_reference_target_file}:/references"
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s
=
input_string
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
reference_scan_list
=
reference_scan_list
,
monitor_column
=
monitor_column
,
roi_path
=
roi_path
,
isolate_spot_by
=
isolate_spot_by
,
signal_path
=
signal_path
,
extracted_reference_target_file
=
extracted_reference_target_file
)
process_input
(
s
,
exploit_slurm_mpi
=
0
)
def
extract_sample_givenrois
(
roi_path
=
None
,
datadir
=
None
,
Start
=
None
,
End
=
None
,
Thickness
=
None
,
monitor_column
=
None
,
signals_target_file
=
None
,
):
for
iE
,
start
in
enumerate
(
range
(
Start
,
End
,
Thickness
)):
end
=
start
+
Thickness
signal_path
=
signals_target_file
+
":/E{iE}"
.
format
(
iE
=
iE
)
input_string
=
"""
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
scan_interval : [{start}, {end}]
energycolumn : sty
signaladdress : {signal_path}
monitorcolumn : {monitor_column}
sumto1D : 0
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
roi_path
=
roi_path
,
start
=
start
,
end
=
end
,
monitor_column
=
monitor_column
,
signal_path
=
signal_path
)
process_input
(
input_string
,
exploit_slurm_mpi
=
0
)
def
resynthetise_scan
(
old_scan_address
=
None
,
response_file
=
None
,
target_address
=
None
,
original_roi_path
=
None
):
input_string
=
"""
superR_recreate_rois :
### we have calculated the responses in responsefilename
### and we want to enlarge the scan by a margin of 3 times
### the original scan on the right and on the left
### ( so for a total of a 7 expansion factor )
responsefilename : {response_file}
nex : 0
## the old scan covered by the old rois
old_scan_address : {old_scan_address}
## where new rois and bnew scan are written
target_filename : {target_address}
filter_rois : 0
original_roi_path : {original_roi_path}
"""
s
=
input_string
.
format
(
response_file
=
response_file
,
target_address
=
target_address
,
old_scan_address
=
old_scan_address
,
original_roi_path
=
original_roi_path
+
"/rois_definition"
)
process_input
(
s
,
exploit_slurm_mpi
=
0
,
stop_omp
=
True
)
def
get_scalars
(
iE
=
None
,
signals_file
=
None
,
reference_file
=
None
,
target_file
=
None
,
selected_rois
=
None
):
inputstring
=
"""
superR_scal_deltaXimages_Esynt :
sample_address : {signals_file}:/E{iE}/scans
delta_address : {reference_file}:/rois_and_reference/scans/ScansSum
load_factors_from :
nbin : 1
target_address : {target_file}:/E{iE}/scal_prods
roi_keys : {selected_rois}
"""
.
format
(
iE
=
iE
,
signals_file
=
signals_file
,
reference_file
=
reference_file
,
target_file
=
target_file
,
selected_rois
=
list
(
selected_rois
)
)
process_input
(
inputstring
,
exploit_slurm_mpi
=
0
)
def
InterpInfo_Esynt_components
(
peaks_shifts
,
signals_file
=
None
,
custom_ene_list
=
None
,
custom_components
=
None
,
selected_rois
=
None
):
f
=
h5py
.
File
(
signals_file
,
"r"
)
volumes_list
=
list
(
f
.
keys
())
ene_list
=
[]
for
v
in
volumes_list
:
dg_Scans
=
[
t
for
t
in
f
[
v
][
"scans"
].
keys
()
if
t
[:
4
]
==
"Scan"
]
ene_list
.
append
(
f
[
v
][
"scans"
][
dg_Scans
[
0
]]
[
"motorDict"
][
"energy"
][()]
)
components
=
h5py
.
File
(
custom_components
,
"r"
)[
"components"
]
[()]
info_dict
=
{}
for
i_interval
in
range
(
len
(
components
)):
info_dict
[
str
(
i_interval
)]
=
{}
info_dict
[
str
(
i_interval
)][
"E"
]
=
custom_ene_list
[
i_interval
]
info_dict
[
str
(
i_interval
)][
"coefficients"
]
=
{}
for
i_n
in
range
(
len
(
energy_exp_grid
)):
info_dict
[
str
(
i_interval
)][
"coefficients"
][
str
(
i_n
)
]
=
{}
for
roi_num
,
de
in
enumerate
(
peaks_shifts
):
info_dict
[
str
(
i_interval
)][
"coefficients"
][
str
(
i_n
)
][
str
(
roi_num
)
]
=
0
for
ic
in
range
(
len
(
components
)):
for
i_interval
in
range
(
len
(
custom_ene_list
)
-
1
):
cE1
=
custom_ene_list
[
i_interval
]
cE2
=
custom_ene_list
[
i_interval
+
1
]
for
i_ene
,
t_ene
in
enumerate
(
energy_exp_grid
)
:
for
roi_num
,
de
in
enumerate
(
peaks_shifts
):
if
selected_rois
and
roi_num
not
in
selected_rois
:
continue
if
t_ene
+
de
<
cE1
or
t_ene
+
de
>
cE2
:
continue
alpha
=
(
cE2
-
(
t_ene
+
de
)
)
/
(
cE2
+
cE1
)
info_dict
[
str
(
ic
)][
"coefficients"
][
str
(
i_ene
)
][
str
(
roi_num
)
]
+=
alpha
*
components
[
ic
][
i_interval
]
info_dict
[
str
(
ic
)][
"coefficients"
][
str
(
i_ene
)
][
str
(
roi_num
)
]
+=
(
1
-
alpha
)
*
components
[
ic
][
i_interval
+
1
]
return
info_dict
def
InterpInfo_Esynt
(
peaks_shifts
,
signals_file
=
None
,
custom_ene_list
=
None
,
selected_rois
=
None
):
f
=
h5py
.
File
(
signals_file
,
"r"
)
volumes_list
=
list
(
f
.
keys
())
ene_list
=
[]
for
v
in
volumes_list
:
dg_Scans
=
[
t
for
t
in
f
[
v
][
"scans"
].
keys
()
if
t
[:
4
]
==
"Scan"
]
ene_list
.
append
(
f
[
v
][
"scans"
][
dg_Scans
[
0
]]
[
"motorDict"
][
"energy"
][()]
)
print
(
" Here custom ene list "
,
custom_ene_list
)
print
(
" Here all the analyser energies "
,
np
.
sort
(
np
.
add
.
outer
(
ene_list
,
peaks_shifts
[
selected_rois
]).
ravel
())
)
order
=
list
(
np
.
argsort
(
ene_list
))
ene_list
=
[
ene_list
[
ii
]
for
ii
in
order
]
volumes_list
=
[
volumes_list
[
ii
]
for
ii
in
order
]
info_dict
=
{}
for
i_intervallo
in
range
(
len
(
custom_ene_list
)):
info_dict
[
str
(
i_intervallo
)]
=
{}
info_dict
[
str
(
i_intervallo
)][
"E"
]
=
custom_ene_list
[
i_intervallo
]
info_dict
[
str
(
i_intervallo
)][
"coefficients"
]
=
{}
for
t_vn
,
t_ene
in
list
(
zip
(
volumes_list
,
ene_list
)):
info_dict
[
str
(
i_intervallo
)][
"coefficients"
][
t_vn
]
=
{}
for
i_intervallo
in
range
(
len
(
custom_ene_list
)
-
1
):
cE1
=
custom_ene_list
[
i_intervallo
]
cE2
=
custom_ene_list
[
i_intervallo
+
1
]
for
t_vn
,
t_ene
in
list
(
zip
(
volumes_list
,
ene_list
))[
0
:]:
for
roi_num
,
de
in
enumerate
(
peaks_shifts
):
if
selected_rois
and
roi_num
not
in
selected_rois
:
continue
if
t_ene
+
de
<
cE1
or
t_ene
+
de
>
cE2
:
continue
alpha
=
(
cE2
-
(
t_ene
+
de
)
)
/
(
cE2
-
cE1
)
info_dict
[
str
(
i_intervallo
)][
"coefficients"
][
str
(
t_vn
)
][
str
(
roi_num
)
]
=
alpha
info_dict
[
str
(
i_intervallo
+
1
)][
"coefficients"
][
str
(
t_vn
)
][
str
(
roi_num
)
]
=
1
-
alpha
return
info_dict
def
tools_sequencer
(
peaks_shifts
=
None
,
datadir
=
None
,
filter_path
=
None
,
roi_scan_num
=
None
,
roi_target_path
=
None
,
first_scan_num
=
None
,
Ydim
=
None
,
Zdim
=
None
,
Edim
=
None
,
monitor_column
=
None
,
signals_target_file
=
None
,
steps_to_do
=
None
,
#####################################################
## can be left to None, will be set to the used target
roi_path
=
None
,
signals_file
=
None
,
reference_clip
=
None
,
isolate_spot_by
=
None
,
reference_scan_list
=
None
,
extracted_reference_target_file
=
None
,
response_target_file
=
None
,
response_fit_options
=
None
,
resynthetised_reference_and_roi_target_file
=
None
,
selected_rois
=
None
,
scalar_products_target_file
=
None
,
volume_retrieval_beta
=
None
,
volume_retrieval_niter
=
None
,
energy_custom_grid
=
None
,
custom_components_file
=
None
,
interpolation_infos_file
=
None
)
:
if
roi_path
is
None
:
roi_path
=
roi_target_path
if
signals_file
is
None
:
signals_file
=
signals_target_file
if
(
steps_to_do
[
"do_step_make_roi"
]):
# ROI selection and reference scan
select_rois
(
datadir
=
datadir
,
roi_scan_num
=
roi_scan_num
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
)
if
(
steps_to_do
[
"do_step_sample_extraction"
]):
extract_sample_givenrois
(
roi_path
=
roi_path
,
datadir
=
datadir
,
Start
=
first_scan_num
,
End
=
(
first_scan_num
+
Zdim
*
Edim
)
,
Thickness
=
Zdim
,
monitor_column
=
monitor_column
,
signals_target_file
=
signals_target_file
)
if
(
steps_to_do
[
"do_step_extract_reference_scan"
]):
# of course we need the REFERENCE SCAN
get_reference
(
datadir
=
datadir
,
roi_path
=
roi_path
,
monitor_column
=
monitor_column
,
extracted_reference_target_file
=
extracted_reference_target_file
,
isolate_spot_by
=
isolate_spot_by
,
reference_scan_list
=
reference_scan_list
)
if
reference_clip
is
not
None
:
clip1
,
clip2
=
reference_clip
ftarget
=
h5py
.
File
(
extracted_reference_target_file
,
"r+"
)
for
roi_scann
in
reference_scan_list
:
target_group
=
ftarget
[
"references/scans/Scan%03d"
%
roi_scann
]
for
k
in
target_group
.
keys
():
if
k
!=
"motorDict"
:
print
(
" SHRINKING scan for ROI %s in file roi_%d.h5 "
%
(
k
,
roi_scann
))
for
dsn
in
[
"matrix"
,
"monitor"
,
"xscale"
]:
mat
=
target_group
[
k
][
dsn
][()]
del
target_group
[
k
][
dsn
]
target_group
[
k
][
dsn
]
=
mat
[
clip1
:
clip2
]
ftarget
.
close
()
ftarget
=
h5py
.
File
(
extracted_reference_target_file
,
"r+"
)
ftarget
[
"references/scans/ScansSum"
]
=
ftarget
[
"references/scans/Scan%03d"
%
reference_scan_list
[
0
]
]
for
other
in
reference_scan_list
[
1
:]:
source_group
=
ftarget
[
"references/scans/Scan%03d"
%
other
]
target_group
=
ftarget
[
"references/scans/ScansSum"
]
for
k
in
target_group
.
keys
():
if
k
!=
"motorDict"
:
print
(
" ADDING data for ROI %s from file roi_%d.h5 "
%
(
k
,
other
))
mat
=
source_group
[
k
][
"matrix"
][()]
target_group
[
k
][
"matrix"
][:]
+=
mat
ftarget
.
close
()
if
(
steps_to_do
[
"do_step_fit_reference_response"
]):
synthetise_response
(
scan_address
=
extracted_reference_target_file
+
":references/scans/ScansSum"
,
target_address
=
response_target_file
+
":/FIT"
,
response_fit_options
=
response_fit_options
)
if
(
steps_to_do
[
"do_step_resynthetise_reference"
]):
resynthetise_scan
(
old_scan_address
=
extracted_reference_target_file
+
":references/scans/ScansSum"
,
response_file
=
response_target_file
+
":/FIT"
,
target_address
=
resynthetised_reference_and_roi_target_file
+
":/rois_and_reference"
,
original_roi_path
=
roi_path
)
if
(
steps_to_do
[
"do_step_scalars"
]):
os
.
system
(
"rm %s"
%
scalar_products_target_file
)
for
iE
in
range
(
Edim
)
:
get_scalars
(
iE
=
iE
,
signals_file
=
signals_file
,
reference_file
=
resynthetised_reference_and_roi_target_file
,
target_file
=
scalar_products_target_file
,
selected_rois
=
selected_rois
)
if
(
steps_to_do
[
"do_step_interpolation_coefficients"
]):
if
custom_components_file
is
None
:
info_dict
=
InterpInfo_Esynt
(
peaks_shifts
,
signals_file
=
signals_file
,
custom_ene_list
=
energy_custom_grid
,
selected_rois
=
selected_rois
)
else
:
info_dict
=
InterpInfo_Esynt_components
(
peaks_shifts
,
signals_file
=
signals_file
,
custom_ene_list
=
energy_custom_grid
,
custom_components
=
custom_components_file
,
selected_rois
=
selected_rois
)
json
.
dump
(
info_dict
,
open
(
interpolation_infos_file
,
"w"
),
indent
=
4
)
if
(
steps_to_do
[
"do_step_finalise_for_fit"
]
):
get_volume_Esynt
(
scalar_products_file
=
scalar_products_target_file
,
interpolation_infos_file
=
interpolation_infos_file
)
def
get_volume_Esynt
(
scalar_products_file
=
None
,
interpolation_infos_file
=
None
):
inputstring
=
"""
superR_getVolume_Esynt :
scalprods_address : {scalar_products_file}:/
dict_interp : {interpolation_infos_file}
output_prefix : DATASFORCC/test0_
"""
.
format
(
scalar_products_file
=
scalar_products_file
,
interpolation_infos_file
=
interpolation_infos_file
)
os
.
system
(
"mkdir DATASFORCC"
)
process_input
(
inputstring
,
exploit_slurm_mpi
=
0
)
XRStools/tools_sequencer_interp.py
0 → 100644
View file @
921ddb77
This diff is collapsed.
Click to expand it.
nonregressions/volumes/esrf_scans/esynth/batch_extraction_esynth1.py
View file @
921ddb77
...
...
@@ -6,8 +6,7 @@ import os
import
h5py
import
math
BATCH_PARALLELISM
=
1
from
XRStools
import
tools_sequencer_esynth
import
os
def
main
():
...
...
@@ -97,7 +96,7 @@ def main():
interpolation_infos_file
=
"interpolation_infos.json"
tools_sequencer
(
peaks_shifts
=
peaks_shifts
,
tools_sequencer_esynth
.
tools_sequencer
(
peaks_shifts
=
peaks_shifts
,
datadir
=
datadir
,
filter_path
=
filter_path
,
roi_scan_num
=
roi_scan_num
,
...
...
@@ -137,457 +136,6 @@ def main():
)
def
process_input
(
s
,
go
=
0
,
exploit_slurm_mpi
=
0
,
stop_omp
=
False
):
open
(
"input_tmp_%d.par"
%
go
,
"w"
).
write
(
s
)
background_activator
=
""
if
(
go
%
BATCH_PARALLELISM
):
background_activator
=
"&"
prefix
=
""
if
stop_omp
:
prefix
=
prefix
+
"export OMP_NUM_THREADS=1 ;"
if
(
exploit_slurm_mpi
==
0
):
os
.
system
(
prefix
+
"mpirun -n 1 XRS_swissknife input_tmp_%d.par %s"
%
(
go
,
background_activator
))
elif
(
exploit_slurm_mpi
>
0
):
os
.
system
(
prefix
+
"mpirun XRS_swissknife input_tmp_%d.par %s"
%
(
go
,
background_activator
)
)
else
:
os
.
system
(
prefix
+
"mpirun -n %d XRS_swissknife input_tmp_%d.par %s"
%
(
abs
(
exploit_slurm_mpi
),
go
,
background_activator
)
)
def
select_rois
(
datadir
=
None
,
roi_scan_num
=
None
,
roi_target_path
=
None
,
filter_path
=
None
):
if
np
.
isscalar
(
roi_scan_num
):
scans
=
[
roi_scan_num
]
else
:
scans
=
list
(
roi_scan_num
)
input_string
=
"""
create_rois:
expdata : {expdata}
scans : {scans}
roiaddress : {roi_target_path}
filter_path : {filter_path}
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
scans
=
scans
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
)
process_input
(
input_string
,
exploit_slurm_mpi
=
0
)
def
get_reference
(
roi_path
=
None
,
datadir
=
None
,
reference_scan_list
=
None
,
monitor_column
=
None
,
extracted_reference_target_file
=
None
,
isolate_spot_by
=
None
):
signal_path
=
extracted_reference_target_file
+
":/"
input_string
=
"""
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitorcolumn : {monitor_column}
scan_list : {reference_scan_list}
signaladdress : "{extracted_reference_target_file}:/references"
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s
=
input_string
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
reference_scan_list
=
reference_scan_list
,
monitor_column
=
monitor_column
,
roi_path
=
roi_path
,
isolate_spot_by
=
isolate_spot_by
,
signal_path
=
signal_path
,
extracted_reference_target_file
=
extracted_reference_target_file
)
process_input
(
s
,
exploit_slurm_mpi
=
0
)
def
extract_sample_givenrois
(
roi_path
=
None
,
datadir
=
None
,
Start
=
None
,
End
=
None
,
Thickness
=
None
,
monitor_column
=
None
,
signals_target_file
=
None
,
):
for
iE
,
start
in
enumerate
(
range
(
Start
,
End
,
Thickness
)):
end
=
start
+
Thickness
signal_path
=
signals_target_file
+
":/E{iE}"
.
format
(
iE
=
iE
)
input_string
=
"""
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
scan_interval : [{start}, {end}]
energycolumn : sty
signaladdress : {signal_path}
monitorcolumn : {monitor_column}
sumto1D : 0
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
roi_path
=
roi_path
,
start
=
start
,
end
=
end
,
monitor_column
=
monitor_column
,
signal_path
=
signal_path
)
process_input
(
input_string
,
exploit_slurm_mpi
=
0
)
def
resynthetise_scan
(
old_scan_address
=
None
,
response_file
=
None
,
target_address
=
None
,
original_roi_path
=
None
):