Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ixstools
xrstools
Commits
00233a02
Commit
00233a02
authored
Dec 01, 2020
by
myron
Browse files
restructuring the volume extraction and xrs_swissknife
parent
23322b8e
Changes
4
Expand all
Hide whitespace changes
Inline
Side-by-side
XRStools/XRS_swissknife.py
View file @
00233a02
This diff is collapsed.
Click to expand it.
XRStools/xrs_read.py
View file @
00233a02
...
...
@@ -2164,7 +2164,7 @@ class read_id20:
self
.
scans
=
{}
# was a dictionary before
if
absfilename
is
not
None
:
if
not
os
.
path
.
isfile
(
absfilename
):
raise
Exception
(
'IOError! No such file, please check filename.'
)
raise
Exception
(
'IOError! No such file
%s
, please check filename.'
%
absfilename
)
self
.
path
=
os
.
path
.
split
(
absfilename
)[
0
]
+
'/'
self
.
filename
=
os
.
path
.
split
(
absfilename
)[
1
]
if
not
edfName
:
...
...
nonregressions/volumes/interpola_1/batch_extraction_esynth1.py
View file @
00233a02
...
...
@@ -3,46 +3,64 @@ import h5py
import
glob
import
json
import
os
import
h5py
BATCH_PARALLELISM
=
4
import
os
def
main
():
peaks_shifts
=
File
(
"peaks_positions_for_analysers.h5"
,
"w"
)[
"peaks_positions"
]
assert
(
len
(
peaks_shifts
)
==
72
))
Enominal
=
np
.
median
(
peak_shifts
)
peak_shifts
-=
Enominal
peaks_shifts
=
h5py
.
File
(
"peaks_positions_for_analysers.h5"
,
"r"
)[
"peaks_positions"
][()]
assert
(
len
(
peaks_shifts
)
==
72
)
os
.
system
(
"xz -dk mask.h5.xz"
)
Enominal
=
np
.
median
(
peaks_shifts
)
peaks_shifts
-=
Enominal
datadir
=
"/data/id20/inhouse/data/run3_20/run3_es949"
os
.
system
(
"mkdir results"
)
roi_target_path
=
"results/myrois.h5:/ROIS"
filter_path
:
"mask.h5:/FILTER_MASK/filter"
roi_scan_num
=
246
# roi_target_path = "rr.h5:/extracted/ROI_AS_SELECTED"
filter_path
=
"mask.h5:/FILTER_MASK/filter"
roi_scan_num
=
[
245
,
246
,
247
]
first_scan_num
=
651
Ydim
=
25
Zdim
=
10
Edim
=
7
monitorcolumn
=
"izero/0.000001"
monitor
_
column
=
"izero/0.000001"
signals_target_file
=
"results/signals.h5"
interpolated_signals_target_file
=
"results/interpolated_signals.h5"
extracted_reference_target_file
=
"results/reference.h5"
reference_scan_list
=
[
245
,
246
,
247
]
# If reference_clip is not None, then a smaller part of the reference scan is considered
# This may be usefule to obtain smaller volumes containing the interesting part
# The used reference scan will the correspond to the positions from reference_clip[0] to reference_clip[1]-1 included
###########
# reference_clip = None
reference_clip
=
[
90
,
180
]
## in the reference scan for each position there is a spot with a maximum. We set zero the background which is further than
## such radius from the maximum
isolate_spot_by
=
6
steps_to_do
=
{
"do_step_make_roi"
:
False
,
"do_step_interpolation"
:
True
,
"do_step_sample_extraction"
:
False
,
"do_step_interpolation"
:
False
,
"do_extract_reference_scan"
:
True
,
"do_step_make_reference"
:
False
,
"do_step_scalar_products"
:
False
,
"do_step_finalise_for_fit"
:
Tru
e
"do_step_finalise_for_fit"
:
Fals
e
}
...
...
@@ -60,8 +78,13 @@ def main():
Edim
=
Edim
,
monitor_column
=
monitor_column
,
signals_target_file
=
signals_target_file
interpolated_signals_target_file
=
interpolated_signals_target_file
signals_target_file
=
signals_target_file
,
interpolated_signals_target_file
=
interpolated_signals_target_file
,
reference_scan_list
=
reference_scan_list
,
reference_clip
=
reference_clip
,
extracted_reference_target_file
=
extracted_reference_target_file
,
isolate_spot_by
=
isolate_spot_by
)
...
...
@@ -87,29 +110,59 @@ def process_input(s, go=0, exploit_slurm_mpi = 0, stop_omp = False):
def
select_rois
(
datadir
=
None
,
roi_scan_num
=
None
,
roi_target_path
=
None
,
filter_path
=
None
):
if
np
.
isscalar
(
roi_scan_num
):
scans
=
[
roi_scan_num
]
else
:
scans
=
list
(
roi_scan_num
)
inputstring
=
"""
create_rois:
expdata : {expdata}
scans :
[{roi_scan_num}]
scans :
{scans}
roiaddress : {roi_target_path}
filter_path : {filter_path}
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"
/
hydra"
),
roi_scan_num
=
roi_scan_num
,
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
scans
=
scans
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
)
process_input
(
inputstring
,
exploit_slurm_mpi
=
0
)
extract_sample_givenrois
(
roi_path
=
roi_path
,
Start
=
first_scan_num
,
End
=
(
first_scan_num
+
Zdim
*
Edim
)
,
Thickness
=
Zdim
)
def
get_reference
(
roi_path
=
None
,
datadir
=
None
,
reference_scan_num
=
None
,
monitor_column
=
None
,
extracted_reference_target_file
=
None
,
isolate_spot_by
=
None
,
):
signal_path
=
extracted_reference_target_file
+
":/"
inputstring
=
"""
loadscan_2Dimages :
expdata : {expdata}
roiaddress : {roi_path}
monitor_column : {monitor_column}
scan_interval : [{reference_scan_num},{reference_scan_num_plus1} ]
signaladdress : {signal_path}
isolateSpot : {isolate_spot_by}
save_also_roi : True
sumto1D : 0
energycolumn : 'stx'
"""
s
=
inputstring
.
format
(
exp_data
=
os
.
path
.
join
(
datadir
,
"hydra"
),
reference_scan_num
=
reference_scan_num
,
reference_scan_num_plus1
=
reference_scan_num
+
1
,
monitor_column
=
monitor_column
,
roi_path
=
roi_path
,
isolate_spot_by
=
isolate_spot_by
,
signal_path
=
signal_path
)
process_input
(
s
,
exploit_slurm_mpi
=
1
)
def
extract_sample_givenrois
(
...
...
@@ -118,8 +171,8 @@ def extract_sample_givenrois(
Start
=
None
,
End
=
None
,
Thickness
=
None
,
monitor_column
s
=
None
,
signals_target_file
=
None
monitor_column
=
None
,
signals_target_file
=
None
,
):
for
start
in
range
(
Start
,
End
,
Thickness
):
...
...
@@ -137,116 +190,112 @@ def extract_sample_givenrois(
monitor_column : {monitor_column}
sumto1D : 0
"""
.
format
(
expdata
=
os
.
path
.
join
(
datadir
,
"
/
hydra"
),
expdata
=
os
.
path
.
join
(
datadir
,
"hydra"
),
roi_path
=
roi_path
,
start
=
start
,
end
=
end
,
monitor_column
=
monitor_column
,
signal_path
=
signal_path
)
process_input
(
s
,
exploit_slurm_mpi
=
1
)
process_input
(
inputstring
,
exploit_slurm_mpi
=
1
)
class
interpolate
(
peaks_shifts
,
interp_file
,
interp_file_target
):
volum_list
=
list
(
interp_file
.
keys
())
scan_num_list
=
np
.
array
([
int
(
t
.
split
(
"_"
)
[
1
])
for
t
in
volum_list
])
ene_list
=
np
.
array
([
interp_file
[
vn
][
"scans"
][
"Scan%03d"
%
sn
][
"motorDict"
][
"energy"
].
value
for
vn
,
sn
in
zip
(
volum_list
,
scan_num_list
)
])
def
interpolate
(
peaks_shifts
,
interp_file_str
,
interp_file_target_str
):
print
(
" ecco la scannumlist "
,
scan_num_list
)
print
(
" ecco ene_list"
,
ene_list
)
interp_file
=
h5py
.
File
(
interp_file_str
,
"r+"
)
interp_file_target
=
h5py
.
File
(
interp_file_target_str
,
"r+"
)
order
=
np
.
argsort
(
ene_list
)
ene_list
=
ene_list
[
order
]
scan_num_list
=
scan_num_list
[
order
]
volum_list
=
[
volum_list
[
ii
]
for
ii
in
order
]
print
(
" ECCO I DATI "
)
print
(
ene_list
)
print
(
cenom
)
# raise
for
t_vn
,
t_sn
,
t_ene
in
list
(
zip
(
volum_list
,
scan_num_list
,
ene_list
))[
0
:]:
rois_coeffs
=
{}
for
roi_num
,
de
in
enumerate
(
cenom
):
print
(
roi_num
,
"===== "
,
t_ene
+
de
,
ene_list
.
min
()
,
t_ene
+
de
,
ene_list
.
max
()
)
if
t_ene
+
de
<
ene_list
.
min
()
or
t_ene
+
de
>
ene_list
.
max
():
continue
print
(
" CONTINUO "
,
t_ene
+
de
,
ene_list
.
min
()
,
ene_list
.
max
()
)
i0
=
np
.
searchsorted
(
ene_list
,
t_ene
+
de
)
-
1
assert
(
i0
>=
0
)
i1
=
i0
+
1
print
(
i0
,
i1
,
len
(
ene_list
))
print
(
ene_list
)
assert
(
i1
<
len
(
ene_list
))
DE
=
(
ene_list
[
i1
]
-
ene_list
[
i0
]
)
df
=
(
t_ene
+
de
-
ene_list
[
i0
]
)
/
DE
rois_coeffs
[
roi_num
]
=
[
i0
,(
1
-
df
)
,
i1
,
df
]
print
(
" for reinterpolation of "
,
t_vn
,
" interpolation scheme is the following "
,
rois_coeffs
)
interp_file_target
.
require_group
(
target
)
fscans
=
interp_file
[
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
volum_list
=
list
(
interp_file
.
keys
())
scan_num_list
=
np
.
array
([
int
(
t
.
split
(
"_"
)
[
1
])
for
t
in
volum_list
])
ene_list
=
np
.
array
([
interp_file
[
vn
][
"scans"
][
"Scan%03d"
%
sn
][
"motorDict"
][
"energy"
].
value
for
vn
,
sn
in
zip
(
volum_list
,
scan_num_list
)
])
# interp_file_target.flush()
fscans
=
interp_file_target
[
target
+
"/"
+
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
print
(
" ecco la scannumlist "
,
scan_num_list
)
print
(
" ecco ene_list"
,
ene_list
)
order
=
np
.
argsort
(
ene_list
)
ene_list
=
ene_list
[
order
]
scan_num_list
=
scan_num_list
[
order
]
volum_list
=
[
volum_list
[
ii
]
for
ii
in
order
]
# raise
for
t_vn
,
t_sn
,
t_ene
in
list
(
zip
(
volum_list
,
scan_num_list
,
ene_list
))[
0
:]:
rois_coeffs
=
{}
for
roi_num
,
de
in
enumerate
(
peaks_shifts
):
print
(
roi_num
,
"===== "
,
t_ene
+
de
,
ene_list
.
min
()
,
t_ene
+
de
,
ene_list
.
max
()
)
if
t_ene
+
de
<
ene_list
.
min
()
or
t_ene
+
de
>
ene_list
.
max
():
continue
print
(
" CONTINUO "
,
t_ene
+
de
,
ene_list
.
min
()
,
ene_list
.
max
()
)
i0
=
np
.
searchsorted
(
ene_list
,
t_ene
+
de
)
-
1
assert
(
i0
>=
0
)
i1
=
i0
+
1
print
(
i0
,
i1
,
len
(
ene_list
))
print
(
ene_list
)
assert
(
i1
<
len
(
ene_list
))
DE
=
(
ene_list
[
i1
]
-
ene_list
[
i0
]
)
df
=
(
t_ene
+
de
-
ene_list
[
i0
]
)
/
DE
rois_coeffs
[
roi_num
]
=
[
i0
,(
1
-
df
)
,
i1
,
df
]
print
(
" for reinterpolation of "
,
t_vn
,
" interpolation scheme is the following "
,
rois_coeffs
)
print
(
" roislist keylist"
,
list
(
rois_coeffs
.
keys
())
)
fscans
=
interp_file
[
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
# interp_file_target.flush()
fscans
=
interp_file_target
[
target
+
"/"
+
t_vn
][
"scans"
]
keys_list
=
list
(
fscans
.
keys
()
)
print
(
" keylist "
,
keys_list
)
print
(
" roislist keylist"
,
list
(
rois_coeffs
.
keys
())
)
for
k
in
keys_list
:
if
k
[:
3
]
==
"ROI"
:
if
int
(
k
[
3
:])
not
in
rois_coeffs
:
print
(
" rimuovo "
,
k
)
del
fscans
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
[:
3
]
==
"ROI"
:
if
int
(
k
[
3
:])
not
in
rois_coeffs
:
print
(
" rimuovo "
,
k
)
del
fscans
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
!=
"motorDict"
:
if
int
(
k
)
not
in
rois_coeffs
:
print
(
" rimuovo da scans"
,
k
)
del
fScan
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
!=
"motorDict"
:
assert
(
int
(
k
)
in
rois_coeffs
)
k
=
int
(
k
)
i0
,
f0
,
i1
,
f1
=
rois_coeffs
[
k
]
matrix0
=
interp_file
[
volum_list
[
i0
]
][
"scans"
][
"Scan%03d"
%
(
scan_num_list
[
i0
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
matrix1
=
interp_file
[
volum_list
[
i1
]
][
"scans"
][
"Scan%03d"
%
(
yyyscan_num_list
[
i1
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
monitor
=
np
.
ones
(
matrix0
.
shape
[
0
],
"f"
)
newmatrix
=
f0
*
matrix0
+
f1
*
matrix1
if
"matrix"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"matrix"
]
if
"monitor"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor"
]
if
"monitor_divider"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor_divider"
]
fScan
[
str
(
k
)][
"matrix"
]
=
newmatrix
fScan
[
str
(
k
)][
"monitor"
]
=
monitor
fScan
[
str
(
k
)][
"monitor_divider"
]
=
1.0
if
k
!=
"motorDict"
:
if
int
(
k
)
not
in
rois_coeffs
:
print
(
" rimuovo da scans"
,
k
)
del
fScan
[
k
]
for
sn
in
range
(
t_sn
,
t_sn
+
1
):
fScan
=
fscans
[
"Scan%03d"
%
sn
]
keys_list
=
list
(
fScan
.
keys
()
)
for
k
in
keys_list
:
if
k
!=
"motorDict"
:
assert
(
int
(
k
)
in
rois_coeffs
)
k
=
int
(
k
)
i0
,
f0
,
i1
,
f1
=
rois_coeffs
[
k
]
matrix0
=
interp_file
[
volum_list
[
i0
]
][
"scans"
][
"Scan%03d"
%
(
scan_num_list
[
i0
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
matrix1
=
interp_file
[
volum_list
[
i1
]
][
"scans"
][
"Scan%03d"
%
(
yyyscan_num_list
[
i1
]
+
sn
-
t_sn
)
][
str
(
k
)][
"matrix"
][:]
monitor
=
np
.
ones
(
matrix0
.
shape
[
0
],
"f"
)
newmatrix
=
f0
*
matrix0
+
f1
*
matrix1
if
"matrix"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"matrix"
]
if
"monitor"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor"
]
if
"monitor_divider"
in
fScan
[
str
(
k
)]
:
del
fScan
[
str
(
k
)][
"monitor_divider"
]
fScan
[
str
(
k
)][
"matrix"
]
=
newmatrix
fScan
[
str
(
k
)][
"monitor"
]
=
monitor
fScan
[
str
(
k
)][
"monitor_divider"
]
=
1.0
...
...
@@ -270,8 +319,14 @@ def tools_sequencer( peaks_shifts = None,
#####################################################
## can be left to None, will be set to the used target
roi_path
=
None
,
signals_file
=
None
interpolated_signals_file
=
None
signals_file
=
None
,
interpolated_signals_file
=
None
,
reference_clip
=
None
,
isolate_spot_by
=
None
,
reference_scan_list
=
None
,
extracted_reference_target_file
=
None
,
)
:
...
...
@@ -286,8 +341,7 @@ def tools_sequencer( peaks_shifts = None,
select_rois
(
datadir
=
datadir
,
roi_scan_num
=
roi_scan_num
,
roi_target_path
=
roi_target_path
,
filter_path
=
filter_path
,
signals_target_file
=
signals_target_file
filter_path
=
filter_path
)
if
(
steps_to_do
[
"do_step_sample_extraction"
]):
...
...
@@ -296,14 +350,62 @@ def tools_sequencer( peaks_shifts = None,
datadir
=
datadir
,
Start
=
first_scan_num
,
End
=
(
first_scan_num
+
Zdim
*
Edim
)
,
Thickness
=
Zdim
Thickness
=
Zdim
,
monitor_column
=
monitor_column
,
signals_target_file
=
signals_target_file
)
if
(
steps_to_do
[
"do_step_interpolation"
]):
os
.
system
(
"cp {signals_file} {interpolated_signals_target_file}"
.
format
(
signals_file
=
signals_file
,
interpolated_signals_target_file
=
interpolated_signals_target_file
)
)
I
nterpolate
(
peaks_shift
,
signals_file
,
interpolated_signals_target_file
)
i
nterpolate
(
peaks_shift
s
,
signals_file
,
interpolated_signals_target_file
)
if
(
steps_to_do
[
"do_extract_reference_scan"
]):
# of course we need the REFERENCE SCAN
for
reference_scan_num
in
reference_scan_list
:
get_reference
(
datadir
=
datadir
,
monitor_column
=
monitor_column
,
extracted_reference_target_file
=
extracted_reference_target_file
,
isolate_spot_by
=
isolate_spot_by
,
reference_scan_num
=
reference_scan_num
)
# for other in other_rois_for_ref:
# os.system("cp roi_%d.h5 roi_%d.h5"%(roi_scann, other) )
# if reference_clip is not None:
# clip1, clip2= reference_clip
# ftarget = h5py.File( "roi_%d.h5" % roi_scann ,"r+")
# target_group = ftarget["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% roi_scann ]
# for k in target_group.keys():
# if k != "motorDict":
# print(" SHRINKING scan for ROI %s in file roi_%d.h5 " %( k, roi_scann ))
# for dsn in ["matrix", "monitor", "xscale"]:
# mat = target_group[k][dsn][()]
# del target_group[k][dsn]
# target_group[k][dsn] = mat[clip1:clip2]
# ftarget.close()
# for other in other_rois_for_ref:
# get_reference(roi_scan_num=other)
# ftarget = h5py.File( "roi_%d.h5" % roi_scann ,"r+")
# fsource = h5py.File( "roi_%d.h5" % other , "r")
# source_group = fsource["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% other ]
# target_group = ftarget["extracted/ROI_AS_SELECTED/calibration_scan/scans/Scan%03d"% roi_scann ]
# for k in target_group.keys():
# if k != "motorDict":
# print(" ADDING data for ROI %s from file roi_%d.h5 " %( k, other ))
# mat = source_group[k]["matrix"][()]
# if clip1 is not None:
# mat = mat[clip1:clip2]
# target_group[k]["matrix"][:] += mat
# print( " SUCCESS ")
main
()
nonregressions/volumes/interpola_1/peaks_positions_for_analysers.h5
0 → 100755
View file @
00233a02
File added
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment