Commit 0e59e3c5 authored by Thomas Vincent's avatar Thomas Vincent

change lzf compression to a configurable on with gzip as default

parent f49d3238
......@@ -46,3 +46,6 @@ class Config(object):
USE_OPENGL = True
"""True to use OpenGL widgets (default), False to disable."""
DEFAULT_HDF5_COMPRESSION = 'gzip'
"""Default HDF5 dataset compression scheme"""
......@@ -37,6 +37,7 @@ from contextlib import contextmanager
import numpy as _np
from .XsocsH5Base import XsocsH5Base
from .. import config
from ..util import text_type
from ._utils import str_to_h5_utf8
......@@ -315,7 +316,7 @@ class QSpaceH5Writer(QSpaceH5):
qspace_shape,
qspace_chunks=None,
qspace_sum_chunks=None,
compression='lzf',
compression='DEFAULT',
coordinates=QSpaceCoordinates.CARTESIAN):
"""Creates empty datasets in the file if not yet created
......@@ -327,6 +328,9 @@ class QSpaceH5Writer(QSpaceH5):
Compression of QSpace and integrated intensity dataset
:param QSpaceCoordinates coordinates: Coordinate system to use
"""
if compression == 'DEFAULT':
compression = config.DEFAULT_HDF5_COMPRESSION
# TODO : mode this to XsocsH5Base ('init_dataset')
if not self.__cube_init:
cube_dtype = _np.float32
......
......@@ -122,7 +122,7 @@ class KmapMerger(object):
self.__spec_h5 = spec_h5
self.__callback = callback
self.__n_proc = config.DEFAULT_PROCESS_NUMBER
self.__compression = 'lzf'
self.__compression = config.DEFAULT_HDF5_COMPRESSION
self.__prefix = 'prefix'
self.__overwrite = False
......
......@@ -31,6 +31,7 @@ __date__ = "01/03/2016"
import os
from ... import config
from . import KmapMerger
from . import KmapSpecParser
......@@ -46,7 +47,7 @@ def merge_scan_data(output_dir,
version=1,
nr_padding=None,
nr_offset=None,
compression='lzf',
compression='DEFAULT',
overwrite=False,
image_roi=None):
"""
......@@ -109,6 +110,10 @@ def merge_scan_data(output_dir,
in the SPEC file.
:type nr_offset: int
:param Union[str,int] compression: The HDF5 compression to use.
:param bool overwrite: True to allow overwriting already existing output file
:param image_roi:
Detector image ROI (origin_row, origin_column, height, width) to save,
or None (default) to save the whole image
......@@ -150,6 +155,8 @@ def merge_scan_data(output_dir,
merger.center_chan = center_chan
merger.chan_per_deg = chan_per_deg
merger.n_proc = n_proc
if compression == 'DEFAULT':
compression = config.DEFAULT_HDF5_COMPRESSION
merger.compression = compression
merger.image_roi = image_roi
......
......@@ -937,7 +937,7 @@ class QSpaceConverter(object):
histo,
selected_entries=entries,
discarded_entries=discarded_entries,
compression='lzf',
compression=config.DEFAULT_HDF5_COMPRESSION,
qspace_chunks=chunks,
qspace_sum_chunks=qspace_sum_chunks,
overwrite=self.__overwrite,
......@@ -1200,7 +1200,7 @@ def _create_result_file(h5_fn,
histo,
selected_entries,
discarded_entries=None,
compression='lzf',
compression='DEFAULT',
qspace_chunks=None,
qspace_sum_chunks=None,
overwrite=False,
......@@ -1253,6 +1253,9 @@ def _create_result_file(h5_fn,
if len(dir_name) > 0 and not os.path.exists(dir_name):
os.makedirs(dir_name)
if compression == 'DEFAULT':
compression = config.DEFAULT_HDF5_COMPRESSION
qspace_h5 = QSpaceH5.QSpaceH5Writer(h5_fn, mode=mode)
qspace_h5.init_file(len(pos_x),
qspace_dims,
......
......@@ -31,7 +31,6 @@ __date__ = "01/11/2016"
import os
import copy
import tempfile
from collections import OrderedDict, namedtuple
......@@ -39,9 +38,7 @@ import numpy as np
import h5py
from ...io.ShiftH5 import ShiftH5Writer
from ...io.XsocsH5Base import XsocsH5Base
from ... import config
ShiftValue = namedtuple('ShiftValue', ['dx', 'dy', 'entry', 'grid_shift'])
......@@ -310,12 +307,13 @@ def _get_distance(xsocs_h5, shifts, ref_p0, ref_p1, tmp_file):
grp['p0'] = ref_p0
grp['p1'] = ref_p1
idx_dset = outh5.require_dataset('data/indices',
(len(entries), n_images),
dtype=np.int32,
shuffle=True,
compression='lzf',
chunks=(1, n_images))
idx_dset = outh5.require_dataset(
'data/indices',
(len(entries), n_images),
dtype=np.int32,
shuffle=True,
compression=config.DEFAULT_HDF5_COMPRESSION,
chunks=(1, n_images))
idx_counts = np.zeros(n_images, dtype=np.int32)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment