Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
X
xsocs
Project overview
Project overview
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
22
Issues
22
List
Boards
Labels
Milestones
JIRA
JIRA
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
kmap
xsocs
Commits
0e59e3c5
Commit
0e59e3c5
authored
Dec 06, 2018
by
Thomas Vincent
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
change lzf compression to a configurable on with gzip as default
parent
f49d3238
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
30 additions
and
15 deletions
+30
-15
xsocs/_config.py
xsocs/_config.py
+3
-0
xsocs/io/QSpaceH5.py
xsocs/io/QSpaceH5.py
+5
-1
xsocs/process/merge/KmapMerger.py
xsocs/process/merge/KmapMerger.py
+1
-1
xsocs/process/merge/helpers.py
xsocs/process/merge/helpers.py
+8
-1
xsocs/process/qspace/QSpaceConverter.py
xsocs/process/qspace/QSpaceConverter.py
+5
-2
xsocs/process/shift/shift.py
xsocs/process/shift/shift.py
+8
-10
No files found.
xsocs/_config.py
View file @
0e59e3c5
...
...
@@ -46,3 +46,6 @@ class Config(object):
USE_OPENGL
=
True
"""True to use OpenGL widgets (default), False to disable."""
DEFAULT_HDF5_COMPRESSION
=
'gzip'
"""Default HDF5 dataset compression scheme"""
xsocs/io/QSpaceH5.py
View file @
0e59e3c5
...
...
@@ -37,6 +37,7 @@ from contextlib import contextmanager
import
numpy
as
_np
from
.XsocsH5Base
import
XsocsH5Base
from
..
import
config
from
..util
import
text_type
from
._utils
import
str_to_h5_utf8
...
...
@@ -315,7 +316,7 @@ class QSpaceH5Writer(QSpaceH5):
qspace_shape
,
qspace_chunks
=
None
,
qspace_sum_chunks
=
None
,
compression
=
'
lzf
'
,
compression
=
'
DEFAULT
'
,
coordinates
=
QSpaceCoordinates
.
CARTESIAN
):
"""Creates empty datasets in the file if not yet created
...
...
@@ -327,6 +328,9 @@ class QSpaceH5Writer(QSpaceH5):
Compression of QSpace and integrated intensity dataset
:param QSpaceCoordinates coordinates: Coordinate system to use
"""
if
compression
==
'DEFAULT'
:
compression
=
config
.
DEFAULT_HDF5_COMPRESSION
# TODO : mode this to XsocsH5Base ('init_dataset')
if
not
self
.
__cube_init
:
cube_dtype
=
_np
.
float32
...
...
xsocs/process/merge/KmapMerger.py
View file @
0e59e3c5
...
...
@@ -122,7 +122,7 @@ class KmapMerger(object):
self
.
__spec_h5
=
spec_h5
self
.
__callback
=
callback
self
.
__n_proc
=
config
.
DEFAULT_PROCESS_NUMBER
self
.
__compression
=
'lzf'
self
.
__compression
=
config
.
DEFAULT_HDF5_COMPRESSION
self
.
__prefix
=
'prefix'
self
.
__overwrite
=
False
...
...
xsocs/process/merge/helpers.py
View file @
0e59e3c5
...
...
@@ -31,6 +31,7 @@ __date__ = "01/03/2016"
import
os
from
...
import
config
from
.
import
KmapMerger
from
.
import
KmapSpecParser
...
...
@@ -46,7 +47,7 @@ def merge_scan_data(output_dir,
version
=
1
,
nr_padding
=
None
,
nr_offset
=
None
,
compression
=
'
lzf
'
,
compression
=
'
DEFAULT
'
,
overwrite
=
False
,
image_roi
=
None
):
"""
...
...
@@ -109,6 +110,10 @@ def merge_scan_data(output_dir,
in the SPEC file.
:type nr_offset: int
:param Union[str,int] compression: The HDF5 compression to use.
:param bool overwrite: True to allow overwriting already existing output file
:param image_roi:
Detector image ROI (origin_row, origin_column, height, width) to save,
or None (default) to save the whole image
...
...
@@ -150,6 +155,8 @@ def merge_scan_data(output_dir,
merger
.
center_chan
=
center_chan
merger
.
chan_per_deg
=
chan_per_deg
merger
.
n_proc
=
n_proc
if
compression
==
'DEFAULT'
:
compression
=
config
.
DEFAULT_HDF5_COMPRESSION
merger
.
compression
=
compression
merger
.
image_roi
=
image_roi
...
...
xsocs/process/qspace/QSpaceConverter.py
View file @
0e59e3c5
...
...
@@ -937,7 +937,7 @@ class QSpaceConverter(object):
histo
,
selected_entries
=
entries
,
discarded_entries
=
discarded_entries
,
compression
=
'lzf'
,
compression
=
config
.
DEFAULT_HDF5_COMPRESSION
,
qspace_chunks
=
chunks
,
qspace_sum_chunks
=
qspace_sum_chunks
,
overwrite
=
self
.
__overwrite
,
...
...
@@ -1200,7 +1200,7 @@ def _create_result_file(h5_fn,
histo
,
selected_entries
,
discarded_entries
=
None
,
compression
=
'
lzf
'
,
compression
=
'
DEFAULT
'
,
qspace_chunks
=
None
,
qspace_sum_chunks
=
None
,
overwrite
=
False
,
...
...
@@ -1253,6 +1253,9 @@ def _create_result_file(h5_fn,
if
len
(
dir_name
)
>
0
and
not
os
.
path
.
exists
(
dir_name
):
os
.
makedirs
(
dir_name
)
if
compression
==
'DEFAULT'
:
compression
=
config
.
DEFAULT_HDF5_COMPRESSION
qspace_h5
=
QSpaceH5
.
QSpaceH5Writer
(
h5_fn
,
mode
=
mode
)
qspace_h5
.
init_file
(
len
(
pos_x
),
qspace_dims
,
...
...
xsocs/process/shift/shift.py
View file @
0e59e3c5
...
...
@@ -31,7 +31,6 @@ __date__ = "01/11/2016"
import
os
import
copy
import
tempfile
from
collections
import
OrderedDict
,
namedtuple
...
...
@@ -39,9 +38,7 @@ import numpy as np
import
h5py
from
...io.ShiftH5
import
ShiftH5Writer
from
...io.XsocsH5Base
import
XsocsH5Base
from
...
import
config
ShiftValue
=
namedtuple
(
'ShiftValue'
,
[
'dx'
,
'dy'
,
'entry'
,
'grid_shift'
])
...
...
@@ -310,12 +307,13 @@ def _get_distance(xsocs_h5, shifts, ref_p0, ref_p1, tmp_file):
grp
[
'p0'
]
=
ref_p0
grp
[
'p1'
]
=
ref_p1
idx_dset
=
outh5
.
require_dataset
(
'data/indices'
,
(
len
(
entries
),
n_images
),
dtype
=
np
.
int32
,
shuffle
=
True
,
compression
=
'lzf'
,
chunks
=
(
1
,
n_images
))
idx_dset
=
outh5
.
require_dataset
(
'data/indices'
,
(
len
(
entries
),
n_images
),
dtype
=
np
.
int32
,
shuffle
=
True
,
compression
=
config
.
DEFAULT_HDF5_COMPRESSION
,
chunks
=
(
1
,
n_images
))
idx_counts
=
np
.
zeros
(
n_images
,
dtype
=
np
.
int32
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment