change lzf compression to a configurable on with gzip as default

xsocs/_config.py

@@ -46,3 +46,6 @@ class Config(object):
 
     USE_OPENGL = True
     """True to use OpenGL widgets (default), False to disable."""
+
+    DEFAULT_HDF5_COMPRESSION = 'gzip'
+    """Default HDF5 dataset compression scheme"""

xsocs/io/QSpaceH5.py

@@ -37,6 +37,7 @@ from contextlib import contextmanager
 import numpy as _np
 
 from .XsocsH5Base import XsocsH5Base
+from .. import config
 from ..util import text_type
 from ._utils import str_to_h5_utf8
 
@@ -315,7 +316,7 @@ class QSpaceH5Writer(QSpaceH5):
                   qspace_shape,
                   qspace_chunks=None,
                   qspace_sum_chunks=None,
-                  compression='lzf',
+                  compression='DEFAULT',
                   coordinates=QSpaceCoordinates.CARTESIAN):
         """Creates empty datasets in the file if not yet created
 
@@ -327,6 +328,9 @@ class QSpaceH5Writer(QSpaceH5):
             Compression of QSpace and integrated intensity dataset
         :param QSpaceCoordinates coordinates: Coordinate system to use
         """
+        if compression == 'DEFAULT':
+            compression = config.DEFAULT_HDF5_COMPRESSION
+
         # TODO : mode this to XsocsH5Base ('init_dataset')
         if not self.__cube_init:
             cube_dtype = _np.float32

xsocs/process/merge/KmapMerger.py

@@ -122,7 +122,7 @@ class KmapMerger(object):
         self.__spec_h5 = spec_h5
         self.__callback = callback
         self.__n_proc = config.DEFAULT_PROCESS_NUMBER
-        self.__compression = 'lzf'
+        self.__compression = config.DEFAULT_HDF5_COMPRESSION
         self.__prefix = 'prefix'
         self.__overwrite = False
 

xsocs/process/merge/helpers.py

@@ -31,6 +31,7 @@ __date__ = "01/03/2016"
 
 import os
 
+from ... import config
 from . import KmapMerger
 from . import KmapSpecParser
 
@@ -46,7 +47,7 @@ def merge_scan_data(output_dir,
                     version=1,
                     nr_padding=None,
                     nr_offset=None,
-                    compression='lzf',
+                    compression='DEFAULT',
                     overwrite=False,
                     image_roi=None):
     """
@@ -109,6 +110,10 @@ def merge_scan_data(output_dir,
         in the SPEC file.
     :type nr_offset: int
 
+    :param Union[str,int] compression: The HDF5 compression to use.
+
+    :param bool overwrite: True to allow overwriting already existing output file
+
     :param image_roi:
         Detector image ROI (origin_row, origin_column, height, width) to save,
         or None (default) to save the whole image
@@ -150,6 +155,8 @@ def merge_scan_data(output_dir,
     merger.center_chan = center_chan
     merger.chan_per_deg = chan_per_deg
     merger.n_proc = n_proc
+    if compression == 'DEFAULT':
+        compression = config.DEFAULT_HDF5_COMPRESSION
     merger.compression = compression
     merger.image_roi = image_roi
 

xsocs/process/qspace/QSpaceConverter.py

@@ -937,7 +937,7 @@ class QSpaceConverter(object):
                                 histo,
                                 selected_entries=entries,
                                 discarded_entries=discarded_entries,
-                                compression='lzf',
+                                compression=config.DEFAULT_HDF5_COMPRESSION,
                                 qspace_chunks=chunks,
                                 qspace_sum_chunks=qspace_sum_chunks,
                                 overwrite=self.__overwrite,
@@ -1200,7 +1200,7 @@ def _create_result_file(h5_fn,
                         histo,
                         selected_entries,
                         discarded_entries=None,
-                        compression='lzf',
+                        compression='DEFAULT',
                         qspace_chunks=None,
                         qspace_sum_chunks=None,
                         overwrite=False,
@@ -1253,6 +1253,9 @@ def _create_result_file(h5_fn,
     if len(dir_name) > 0 and not os.path.exists(dir_name):
         os.makedirs(dir_name)
 
+    if compression == 'DEFAULT':
+        compression = config.DEFAULT_HDF5_COMPRESSION
+
     qspace_h5 = QSpaceH5.QSpaceH5Writer(h5_fn, mode=mode)
     qspace_h5.init_file(len(pos_x),
                         qspace_dims,

xsocs/process/shift/shift.py

@@ -31,7 +31,6 @@ __date__ = "01/11/2016"
 
 
 import os
-import copy
 import tempfile
 from collections import OrderedDict, namedtuple
 
@@ -39,9 +38,7 @@ import numpy as np
 
 import h5py
 
-
-from ...io.ShiftH5 import ShiftH5Writer
-from ...io.XsocsH5Base import XsocsH5Base
+from ... import config
 
 
 ShiftValue = namedtuple('ShiftValue', ['dx', 'dy', 'entry', 'grid_shift'])
@@ -310,12 +307,13 @@ def _get_distance(xsocs_h5, shifts, ref_p0, ref_p1, tmp_file):
         grp['p0'] = ref_p0
         grp['p1'] = ref_p1
 
-        idx_dset = outh5.require_dataset('data/indices',
-                                         (len(entries), n_images),
-                                         dtype=np.int32,
-                                         shuffle=True,
-                                         compression='lzf',
-                                         chunks=(1, n_images))
+        idx_dset = outh5.require_dataset(
+            'data/indices',
+            (len(entries), n_images),
+            dtype=np.int32,
+            shuffle=True,
+            compression=config.DEFAULT_HDF5_COMPRESSION,
+            chunks=(1, n_images))
 
         idx_counts = np.zeros(n_images, dtype=np.int32)