Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ixstools
xrstools
Commits
00233a02
Commit
00233a02
authored
Dec 01, 2020
by
myron
Browse files
restructuring the volume extraction and xrs_swissknife
parent
23322b8e
Changes
4
Expand all
Hide whitespace changes
Inline
Side-by-side
XRStools/XRS_swissknife.py
View file @
00233a02
This diff is collapsed.
Click to expand it.
XRStools/xrs_read.py
View file @
00233a02
...
...
@@ -2164,7 +2164,7 @@ class read_id20:
self
.
scans
=
{}
# was a dictionary before
if
absfilename
is
not
None
:
if
not
os
.
path
.
isfile
(
absfilename
):
raise
Exception
(
'IOError! No such file, please check filename.'
)
raise
Exception
(
'IOError! No such file
%s
, please check filename.'
%
absfilename
)
self
.
path
=
os
.
path
.
split
(
absfilename
)[
0
]
+
'/'
self
.
filename
=
os
.
path
.
split
(
absfilename
)[
1
]
if
not
edfName
:
...
...
nonregressions/volumes/interpola_1/batch_extraction_esynth1.py
View file @
00233a02
...
...
@@ -3,46 +3,64 @@ import h5py
import
glob
import
json
import
os
import
h5py
BATCH_PARALLELISM
=
4
import
os
def
main
():
peaks_shifts
=
File
(
"peaks_positions_for_analysers.h5"
,
"w"
)[
"peaks_positions"
]
assert
(
len
(
peaks_shifts
)
==
72
))
Enominal
=
np
.
median
(
peak_shifts
)
peak_shifts
-=
Enominal
peaks_shifts
=
h5py
.
File
(
"peaks_positions_for_analysers.h5"
,
"r"
)[
"peaks_positions"
][()]
assert
(
len
(
peaks_shifts
)
==
72
)
os
.
system
(
"xz -dk mask.h5.xz"
)
Enominal
=
np
.
median
(
peaks_shifts
)
peaks_shifts
-=
Enominal
datadir
=
"/data/id20/inhouse/data/run3_20/run3_es949"
os
.
system
(
"mkdir results"
)
roi_target_path
=
"results/myrois.h5:/ROIS"
filter_path
:
"mask.h5:/FILTER_MASK/filter"
roi_scan_num
=
246
# roi_target_path = "rr.h5:/extracted/ROI_AS_SELECTED"
filter_path
=
"mask.h5:/FILTER_MASK/filter"
roi_scan_num
=
[
245
,
246
,
247
]
first_scan_num
=
651
Ydim
=
25
Zdim
=
10
Edim
=
7
monitorcolumn
=
"izero/0.000001"
monitor
_
column
=
"izero/0.000001"
signals_target_file
=
"results/signals.h5"
interpolated_signals_target_file
=
"results/interpolated_signals.h5"
extracted_reference_target_file
=
"results/reference.h5"
reference_scan_list
=
[
245
,
246
,
247
]
# If reference_clip is not None, then a smaller part of the reference scan is considered
# This may be usefule to obtain smaller volumes containing the interesting part
# The used reference scan will the correspond to the positions from reference_clip[0] to reference_clip[1]-1 included
###########
# reference_clip = None
reference_clip
=
[
90
,
180
]
## in the reference scan for each position there is a spot with a maximum. We set zero the background which is further than
## such radius from the maximum
isolate_spot_by
=
6
steps_to_do
=
{
"do_step_make_roi"
:
False
,
"do_step_interpolation"
:
True
,
"do_step_sample_extraction"
:
False
,
"do_step_interpolation"
:
False
,
"do_extract_reference_scan"
:
True
,
"do_step_make_reference"
:
False
,
"do_step_scalar_products"
:
False
,
"do_step_finalise_for_fit"
:
Tru
e
"do_step_finalise_for_fit"
:
Fals
e
}
...
...
@@ -60,8 +78,13 @@ def main():
Edim
=
Edim
,
monitor_column
=
monitor_column
,
signals_target_file
=
signals_target_file
interpolated_signals_target_file
=
interpolated_signals_target_file
signals_target_file
=
signals_target_file
,
interpolated_signals_target_file
=
interpolated_signals_target_file
,
reference_scan_list
=
reference_scan_list
,
reference_clip
=
reference_clip
,
extracted_reference_target_file
=
extracted_reference_target_file
,
isolate_spot_by
=
isolate_spot_by
)
...
...
@@ -87,29 +110,59 @@ def process_input(s, go=0, exploit_slurm_mpi = 0, stop_omp = False):
def
select_rois
(
datadir
=
None
,
roi_scan_num
=
None
,
roi_target_path
=
None
,
filter_path
=
None
):
if
np
.
isscalar
(
roi_scan_num
):
scans
=
[
roi_scan_num
]
else
:
scans
=
list
(
roi_scan_num
)
inputstring
=
"""
create_rois:
expdata : {expdata}
scans :
[{roi_scan_num}]
scans :
{scans}
roiaddress : {roi_target_path}
filter_path : {filter_path}
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"
/
hydra"
),
roi_scan_num
=
roi_scan_num
,
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
scans
=
scans
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
)
process_input
(
inputstring
,
exploit_slurm_mpi
=
0
)
extract_sample_givenrois
(
roi_path
=
roi_path
,
Start
=
first_scan_num
,
End
=
(
first_scan_num
+
Zdim
*
Edim
)
,
Thickness
=
Zdim
)
def
get_reference
(
roi_path
=
None
,
datadir
=
None
,
reference_scan_num
=
None
,
monitor_column
=
None
,
extracted_reference_target_file
=
None
,
isolate_spot_by
=
None
,
):
signal_path
=
extracted_reference_target_file
+
":/"
inputstring
=
"""
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitor_column : {monitor_column}
scan_interval : [{reference_scan_num},{reference_scan_num_plus1} ]
signaladdress : {signal_path}
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s
=
inputstring
.
format
(
exp_data
=
os
.
path
.
join
(
datadir
,
"hydra"
),
reference_scan_num
=
reference_scan_num
,
reference_scan_num_plus1
=
reference_scan_num
+
1
,
monitor_column
=
monitor_column
,
roi_path
=
roi_path
,
isolate_spot_by
=
isolate_spot_by
,
signal_path
=
signal_path
)
process_input
(
s
,
exploit_slurm_mpi
=
1
)
def
extract_sample_givenrois
(
...
...
@@ -118,8 +171,8 @@ def extract_sample_givenrois(
Start
=
None
,
End
=
None
,
Thickness
=
None
,
monitor_column
s
=
None
,
signals_target_file
=
None
monitor_column
=
None
,
signals_target_file
=
None
,
):
for
start
in
range
(
Start
,
End
,
Thickness
):
...
...
@@ -137,116 +190,112 @@ def extract_sample_givenrois(
monitor_column : {monitor_column}
sumto1D : 0
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"
/
hydra"
),
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
roi_path
=
roi_path
,
start
=
start
,
end
=
end
,
monitor_column
=
monitor_column
,
signal_path
=
signal_path
)
process_input
(
s
,
exploit_slurm_mpi
=
1
)
process_input
(
inputstring
,
exploit_slurm_mpi
=
1
)
class
interpolate
(
peaks_shifts
,
interp_file
,
interp_file_target
):
volum_list
=
list
(
interp_file
.
keys
())
scan_num_list
=
np
.
array
([
int
(
t
.
split
(
"_"
)
[
1
])
for
t
in
volum_list
])
ene_list
=
np
.
array
([
interp_file
[
vn
][
"scans"
][
"Scan%03d"
%
sn
][
"motorDict"
][
"energy"
].
value
for
vn
,
sn
in
zip
(
volum_list
,
scan_num_list
)
])
def
interpolate
(
peaks_shifts
,
interp_file_str
,
interp_file_target_str
):
print
(
" ecco la scannumlist "
,
scan_num_list
)
print
(
" ecco ene_list"
,
ene_list
)
interp_file
=
h5py
.
File
(
interp_file_str
,
"r+"
)
interp_file_target
=
h5py
.
File
(
interp_file_target_str
,
"r+"
)
order
=
np
.
argsort
(
ene_list
)
ene_list
=
ene_list
[
order
]
scan_num_list
=
scan_num_list
[
order
]
volum_list
=
[
volum_list
[
ii
]
for
ii
in
order
]
print
(
" ECCO I DATI "
)
print
(
ene_list
)
print
(
cenom
)
# raise
for
t_vn
,
t_sn
,
t_ene
in
list
(
zip
(
volum_list
,
scan_num_list
,
ene_list
))[
0
:]:
rois_coeffs
=
{}
for
roi_num
,
de
in
enumerate
(
cenom
):
print
(
roi_num
,
"===== "
,
t_ene
+
de
,
ene_list
.
min
()
,
t_ene
+
de
,
ene_list
.
max
()
)
if
t_ene
+
de
<
ene_list
.
min
()
or
t_ene
+
de
>
ene_list
.
max
():
continue
print
(
" CONTINUO "
,
t_ene
+
de
,
ene_list
.
min
()
,
ene_list
.
max
()
)
i0
=
np
.
searchsorted
(
ene_list
,
t_ene
+
de
)
-
1
assert
(
i0
>=
0
)
i1
=
i0
+
1
print
(
i0
,
i1
,
len
(
ene_list
))
print
(
ene_list
)
assert
(
i1
<
len
(
ene_list
))
DE
=
(
ene_list
[
i1
]
-
ene_list
[
i0
]
)
df
=
(
t_ene
+
de
-
ene_list
[
i0
]
)
/
DE
rois_coeffs
[
roi_num
]
=
[
i0
,(
1
-
df
)
,
i1
,
df
]
print
(
" for reinterpolation of "
,
t_vn
,
" interpolation scheme is the following "
,
rois_coeffs
)
interp_file_target
.
require_group
(
target
)
fscans
=
interp_file
[
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
volum_list
=
list
(
interp_file
.
keys
())
scan_num_list
=
np
.
array
([
int
(
t
.
split
(
"_"
)
[
1
])
for
t
in
volum_list
])
ene_list
=
np
.
array
([
interp_file
[
vn
][
"scans"
][
"Scan%03d"
%
sn
][
"motorDict"
][
"energy"
].
value
for
vn
,
sn
in
zip
(
volum_list
,
scan_num_list
)
])
# interp_file_target.flush()
fscans
=
interp_file_target
[
target
+
"/"
+
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
print
(
" ecco la scannumlist "
,
scan_num_list
)
print
(
" ecco ene_list"
,
ene_list
)
order
=
np
.
argsort
(
ene_list
)
ene_list
=
ene_list
[
order
]
scan_num_list
=
scan_num_list
[
order
]
volum_list
=
[
volum_list
[
ii
]
for
ii
in
order
]
# raise
for
t_vn
,
t_sn
,
t_ene
in
list
(
zip
(
volum_list
,
scan_num_list
,
ene_list
))[
0
:]:
rois_coeffs
=
{}
for
roi_num
,
de
in
enumerate
(
peaks_shifts
):
print
(
roi_num
,
"===== "
,
t_ene
+
de
,
ene_list
.
min
()
,
t_ene
+
de
,
ene_list
.
max
()
)
if
t_ene
+
de
<
ene_list
.
min
()
or
t_ene
+
de
>
ene_list
.
max
():
continue
print
(
" CONTINUO "
,
t_ene
+
de
,
ene_list
.
min
()
,
ene_list
.
max
()
)
i0
=
np
.
searchsorted
(
ene_list
,
t_ene
+
de
)
-
1
assert
(
i0
>=
0
)
i1
=
i0
+
1
print
(
i0
,
i1
,
len
(
ene_list
))
print
(
ene_list
)
assert
(
i1
<
len
(
ene_list
))
DE
=
(
ene_list
[
i1
]
-
ene_list
[
i0
]
)
df
=
(
t_ene
+
de
-
ene_list
[
i0
]
)
/
DE
rois_coeffs
[
roi_num
]
=
[
i0
,(
1
-
df
)
,
i1
,
df
]
print
(
" for reinterpolation of "
,
t_vn
,
" interpolation scheme is the following "
,
rois_coeffs
)
print
(
" roislist keylist"
,
list
(
rois_coeffs
.
keys
())
)
fscans
=
interp_file
[
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
# interp_file_target.flush()
fscans
=
interp_file_target
[
target
+
"/"
+
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
print
(
" roislist keylist"
,
list
(
rois_coeffs
.
keys
())
)
for
k
in
keys_list
:
if
k
[:
3
]
==
"ROI"
:
if
int
(
k
[
3
:])
not
in
rois_coeffs
:
print
(
" rimuovo "
,
k
)
del
fscans
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
[:
3
]
==
"ROI"
:
if
int
(
k
[
3
:])
not
in
rois_coeffs
:
print
(
" rimuovo "
,
k
)
del
fscans
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
!=
"motorDict"
:
if
int
(
k
)
not
in
rois_coeffs
:
print
(
" rimuovo da scans"
,
k
)
del
fScan
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
!=
"motorDict"
:
assert
(
int
(
k
)
in
rois_coeffs
)
k
=
int
(
k
)
i0
,
f0
,
i1
,
f1
=
rois_coeffs
[
k
]
matrix0
=
interp_file
[
volum_list
[
i0
]
][
"scans"
][
"Scan%03d"
%
(
scan_num_list
[
i0
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
matrix1
=
interp_file
[
volum_list
[
i1
]
][
"scans"
][
"Scan%03d"
%
(
yyyscan_num_list
[
i1
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
monitor
=
np
.
ones
(
matrix0
.
shape
[
0
],
"f"
)
newmatrix
=
f0
*
matrix0
+
f1
*
matrix1
if
"matrix"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"matrix"
]
if
"monitor"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor"
]
if
"monitor_divider"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor_divider"
]
fScan
[
str
(
k
)][
"matrix"
]
=
newmatrix
fScan
[
str
(
k
)][
"monitor"
]
=
monitor
fScan
[
str
(
k
)][
"monitor_divider"
]
=
1.0
if
k
!=
"motorDict"
:
if
int
(
k
)
not
in
rois_coeffs
:
print
(
" rimuovo da scans"
,
k
)
del
fScan
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
!=
"motorDict"
:
assert
(
int
(
k
)
in
rois_coeffs
)
k
=
int
(
k
)
i0
,
f0
,
i1
,
f1
=
rois_coeffs
[
k
]
matrix0
=
interp_file
[
volum_list
[
i0
]
][
"scans"
][
"Scan%03d"
%
(
scan_num_list
[
i0
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
matrix1
=
interp_file
[
volum_list
[
i1
]
][
"scans"
][
"Scan%03d"
%
(
yyyscan_num_list
[
i1
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
monitor
=
np
.
ones
(
matrix0
.
shape
[
0
],
"f"
)
newmatrix
=
f0
*
matrix0
+
f1
*
matrix1
if
"matrix"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"matrix"
]
if
"monitor"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor"
]
if
"monitor_divider"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor_divider"
]
fScan
[
str
(
k
)][
"matrix"
]
=
newmatrix
fScan
[
str
(
k
)][
"monitor"
]
=
monitor
fScan
[
str
(
k
)][
"monitor_divider"
]
=
1.0
...
...
@@ -270,8 +319,14 @@ def tools_sequencer( peaks_shifts = None,
#####################################################
## can be left to None, will be set to the used target
roi_path
=
None
,
signals_file
=
None
interpolated_signals_file
=
None
signals_file
=
None
,
interpolated_signals_file
=
None
,
reference_clip
=
None
,
isolate_spot_by
=
None
,
reference_scan_list
=
None
,
extracted_reference_target_file
=
None
,
)
:
...
...
@@ -286,8 +341,7 @@ def tools_sequencer( peaks_shifts = None,
select_rois
(
datadir
=
datadir
,
roi_scan_num
=
roi_scan_num
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
,
signals_target_file
=
signals_target_file
filter_path
=
filter_path
)
if
(
steps_to_do
[
"do_step_sample_extraction"
]):
...
...
@@ -296,14 +350,62 @@ def tools_sequencer( peaks_shifts = None,
datadir
=
datadir
,
Start
=
first_scan_num
,
End
=
(
first_scan_num
+
Zdim
*
Edim
)
,
Thickness
=
Zdim
Thickness
=
Zdim
,
monitor_column
=
monitor_column
,
signals_target_file
=
signals_target_file
)
if
(
steps_to_do
[
"do_step_interpolation"
]):
os
.
system
(
"cp {signals_file} {interpolated_signals_target_file}"
.
format
(
signals_file
=
signals_file
,
interpolated_signals_target_file
=
interpolated_signals_target_file
)
)
I
nterpolate
(
peaks_shift
,
signals_file
,
interpolated_signals_target_file
)
i
nterpolate
(
peaks_shift
s
,
signals_file
,
interpolated_signals_target_file
)
if
(
steps_to_do
[
"do_extract_reference_scan"
]):
# of course we need the REFERENCE SCAN
for
reference_scan_num
in
reference_scan_list
:
get_reference
(
datadir
=
datadir
,
monitor_column
=
monitor_column
,
extracted_reference_target_file
=
extracted_reference_target_file
,
isolate_spot_by
=
isolate_spot_by
,
reference_scan_num
=
reference_scan_num
)
# for other in other_rois_for_ref:
# os.system("cp roi_%d.h5 roi_%d.h5"%(roi_scann, other) )
# if reference_clip is not None:
# clip1, clip2= reference_clip
# ftarget = h5py.File( "roi_%d.h5" % roi_scann ,"r+")
# target_group = ftarget["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% roi_scann ]
# for k in target_group.keys():
# if k != "motorDict":
# print(" SHRINKING scan for ROI %s in file roi_%d.h5 " %( k, roi_scann ))
# for dsn in ["matrix", "monitor", "xscale"]:
# mat = target_group[k][dsn][()]
# del target_group[k][dsn]
# target_group[k][dsn] = mat[clip1:clip2]
# ftarget.close()
# for other in other_rois_for_ref:
# get_reference(roi_scan_num=other)
# ftarget = h5py.File( "roi_%d.h5" % roi_scann ,"r+")
# fsource = h5py.File( "roi_%d.h5" % other , "r")
# source_group = fsource["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% other ]
# target_group = ftarget["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% roi_scann ]
# for k in target_group.keys():
# if k != "motorDict":
# print(" ADDING data for ROI %s from file roi_%d.h5 " %( k, other ))
# mat = source_group[k]["matrix"][()]
# if clip1 is not None:
# mat = mat[clip1:clip2]
# target_group[k]["matrix"][:] += mat
# print( " SUCCESS ")
main
()
nonregressions/volumes/interpola_1/peaks_positions_for_analysers.h5
0 → 100755
View file @
00233a02
File added
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment