Merge branch 'misc_2021.1.0' into 'master'

Miscellaneous fixes/improvements

Closes #245, #138, #231, #191, and #202

See merge request !134

CHANGELOG.md

 # Change Log
 
+## 2021.1.1
+
+### Application
+
+   - Added `unsharp_method` to select unsharp mask method (gaussian or log)
+   - `enable_halftomo` can now be set to `auto` to let nabu determine if half aqcuisition was used
+
+### Library
+
+   - `scipy` is now a mandatory dependency
+
+
 ## 2021.1.0
 
 ### Application

doc/features.md

@@ -98,10 +98,10 @@ API: [MunchDeringer](apidoc/nabu.preproc.rings) and [CudaMunchDeringer](apidoc/n
 ### Flats distortion correction
 
 A flats distortion estimation/correction is available. If activated, each radio is correlated with its corresponding flat, in order to determine and correct the flat distortion.  
-  
+
 ```{warning}
 This is currently quite slow (many correlations), and only supported by the "full radios pipeline"
-``` 
+```
 
 Configuration file: `[preproc]`: `flat_distortion_params` and `flat_distortion_correction_enabled`
 
@@ -145,7 +145,7 @@ Although it is a general-purpose image processing utility, unsharp mask is often
 
 Each radio is processed as `UnsharpedImage = (1 + coeff)*Image - coeff * ConvolvedImage` where `ConvolvedImage` is the result of a Gaussian blur applied on the image.
 
-Configuration file: section `[phase]`: `unsharp_coeff = 1.`  and `unsharp_sigma = 1.`
+Configuration file: section `[phase]`: `unsharp_coeff = 1.`, `unsharp_sigma = 1.`, `unsharp_method = gaussian`.
 
 Setting `coeff` to zero (default) disables unsharp masking.
 

nabu/app/fullfield.py

@@ -559,7 +559,7 @@ class FullFieldPipeline:
         self.unsharp_mask = self.UnsharpMaskClass(
             self._get_shape("unsharp_mask"),
             options["unsharp_sigma"], options["unsharp_coeff"],
-            mode="reflect", method="gaussian"
+            mode="reflect", method=options["unsharp_method"]
         )
         self.register_callback("unsharp_mask", FullFieldPipeline._reshape_radios_after_phase)
 

nabu/io/config.py

@@ -153,6 +153,7 @@ def validate_nabu_config(config):
     if isinstance(config, str):
         config = NabuConfigParser(config).conf_dict
     res_config = {}
+    to_remove = []
     for section, section_content in config.items():
         # Ignore the "other" section
         if section.lower() == "other":
@@ -168,20 +169,27 @@ def validate_nabu_config(config):
                 continue # deleted key
             if opt is None:
                 raise ValueError("Unknown option '%s' in section [%s]" % (key, section_updated))
+            if nabu_config[section_updated][key]["type"] == "unsupported":
+                to_remove.append((section_updated, key))
+                continue
             validator = nabu_config[section_updated][key]["validator"]
             if section_updated not in res_config: # missing section - handled later
                 continue
             res_config[section_updated][key] = validator(section_updated, key, value)
+    # Remove 'unsupported' params
+    for param in to_remove:
+        res_config[param[0]].pop(param[1])
     # Handle sections missing in config
     for section in (set(nabu_config.keys()) - set(res_config.keys())):
         res_config[section] = _extract_nabuconfig_section(section)
         for key, value in res_config[section].items():
+            if nabu_config[section][key]["type"] == "unsupported":
+                continue
             validator = nabu_config[section][key]["validator"]
             res_config[section][key] = validator(section, key, value)
     return res_config
 
 
-
 def convert_dict_values(dic, val_replacements, bytes_tostring=False):
     """
     Modify a dictionary to be able to export it with silx.io.dicttoh5
@@ -220,6 +228,10 @@ def export_dict_to_h5(dic, h5file, h5path, overwrite_data=True, mode="a"):
     mode: str, optional
         File mode. Default is "a" (append).
     """
+    # in silx >= 0.15, overwrite_data = True becomes update_mode='modify
+    # and overwrite_data=False becomes update_mode='add'
+    update_mode = ["add", "modify"][overwrite_data]
+    #
     modified_dic = convert_dict_values(
         dic,
         {None: "None"},
@@ -228,7 +240,7 @@ def export_dict_to_h5(dic, h5file, h5path, overwrite_data=True, mode="a"):
         modified_dic,
         h5file=h5file,
         h5path=h5path,
-        overwrite_data=overwrite_data,
+        update_mode=update_mode,
         mode=mode
     )
 

nabu/resources/nabu_config.py

@@ -196,6 +196,12 @@ nabu_config = {
             "validator": float_validator,
             "type": "optional",
         },
+        "unsharp_method": {
+            "default": "gaussian",
+            "help": "Which type of unsharp mask filter to use. Available values are gaussian (UnsharpedImage =  (1 + coeff)*originalPaganinImage - coeff * ConvolvedImage) and laplacian (UnsharpedImage = originalPaganinImage + coeff * ConvolvedImage). Default is gaussian.",
+            "validator": unsharp_method_validator,
+            "type": "optional",
+        },
         "padding_type": {
             "default": "edge",
             "help": "Padding type for the filtering step in Paganin/CTF. Available are: mirror, edge, zeros",
@@ -283,9 +289,9 @@ nabu_config = {
             "type": "optional", # put "advanced" with default value "edges" ?
         },
         "enable_halftomo": {
-            "default": "0",
-            "help": "Whether to enable half-acquisition",
-            "validator": boolean_validator,
+            "default": "auto",
+            "help": "Whether to enable half-acquisition. Default is auto. You can enable/disable it manually by setting 1 or 0.",
+            "validator": boolean_or_auto_validator,
             "type": "optional",
         },
         "start_x": {
@@ -406,13 +412,13 @@ nabu_config = {
             "default": "1",
             "help": "Number of GPUs to use.",
             "validator": nonnegative_integer_validator,
-            "type": "unsupported",
+            "type": "advanced",
         },
         "gpu_id": {
             "default": "",
             "help": "For method = local only. List of GPU IDs to use. This parameter overwrites 'gpus'.\nIf left blank, exactly one GPU will be used, and the best one will be picked.",
             "validator": list_of_int_validator,
-            "type": "unsupported",
+            "type": "advanced",
         },
         "cpu_workers": {
             "default": "0",

nabu/resources/params.py

@@ -15,6 +15,14 @@ phase_retrieval_methods = {
     "ctf": "CTF",
 }
 
+unsharp_methods = {
+    "gaussian": "gaussian",
+    "log": "log",
+    "laplacian": "log",
+    "none": None,
+    "": None,
+}
+
 padding_modes = {
     "edges": "edge",
     "edge": "edge",

nabu/resources/processconfig.py

@@ -115,27 +115,28 @@ class ProcessConfig:
 
 
     def _get_cor(self):
-        cor = self.nabu_config["reconstruction"]["rotation_axis_position"]
+        rec_config = self.nabu_config["reconstruction"]
+        cor = rec_config["rotation_axis_position"]
         if isinstance(cor, str): # auto-CoR
-            cor_slice = self.nabu_config["reconstruction"]["cor_slice"]
+            cor_slice = rec_config["cor_slice"]
             if cor_slice is not None or cor == "sino-coarse-to-fine":
                 subsampling = extract_parameters(
-                    self.nabu_config["reconstruction"]["cor_options"]
+                    rec_config["cor_options"]
                 ).get("subsampling", 10)
                 self.corfinder = SinoCOREstimator(
                     self.dataset_infos,
                     cor_slice or 0,
                     subsampling=subsampling,
                     do_flatfield=self.nabu_config["preproc"]["flatfield_enabled"],
-                    cor_options=self.nabu_config["reconstruction"]["cor_options"],
+                    cor_options=rec_config["cor_options"],
                     logger=self.logger
                 )
             else:
                 self.corfinder = COREstimator(
                     self.dataset_infos,
-                    halftomo=self.nabu_config["reconstruction"]["enable_halftomo"],
+                    halftomo=self.do_halftomo,
                     do_flatfield=self.nabu_config["preproc"]["flatfield_enabled"],
-                    cor_options=self.nabu_config["reconstruction"]["cor_options"],
+                    cor_options=rec_config["cor_options"],
                     logger=self.logger
                 )
             cor = self.corfinder.find_cor(method=cor)
@@ -162,6 +163,20 @@ class ProcessConfig:
         self.dataset_infos.detector_tilt = tilt
 
 
+    @property
+    def do_halftomo(self):
+        """
+        Return True if the current dataset is to be reconstructed using 'half-acquisition' setting.
+        """
+        enable_halftomo = self.nabu_config["reconstruction"]["enable_halftomo"]
+        is_halftomo_dataset = self.dataset_infos.is_halftomo
+        if enable_halftomo == "auto":
+            if is_halftomo_dataset is None:
+                raise ValueError("enable_halftomo was set to 'auto', but information on field of view was not found. Please set either 0 or 1 for enable_halftomo")
+            return is_halftomo_dataset
+        return enable_halftomo
+
+
     def validation_stage2(self):
         validator = NabuValidator(self.nabu_config, self.dataset_infos)
         if self.checks:
@@ -182,7 +197,7 @@ class ProcessConfig:
         phase_method = self.nabu_config["phase"]["method"]
         do_ctf = phase_method == "CTF"
         do_pag = phase_method == "paganin"
-        do_unsharp = self.nabu_config["phase"]["unsharp_coeff"] > 0
+        do_unsharp = self.nabu_config["phase"]["unsharp_method"] is not None and self.nabu_config["phase"]["unsharp_coeff"] > 0
         if user_rotate_projections is None and tilt is None:
             return None
         if do_ctf:
@@ -286,10 +301,10 @@ class ProcessConfig:
         #
         # Unsharp
         #
-        if nabu_config["phase"]["unsharp_coeff"] > 0:
+        if nabu_config["phase"]["unsharp_method"] is not None and nabu_config["phase"]["unsharp_coeff"] > 0:
             tasks.append("unsharp_mask")
             options["unsharp_mask"] = copy_dict_items(
-                nabu_config["phase"], ["unsharp_coeff", "unsharp_sigma"]
+                nabu_config["phase"], ["unsharp_coeff", "unsharp_sigma", "unsharp_method"]
             )
         #
         # -logarithm
@@ -339,8 +354,7 @@ class ProcessConfig:
             tasks.append("build_sino")
             options["build_sino"] = copy_dict_items(
                 nabu_config["reconstruction"],
-                ["rotation_axis_position", "enable_halftomo", "start_x", "end_x",
-                 "start_y", "end_y", "start_z", "end_z"]
+                ["rotation_axis_position", "start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
             )
             options["build_sino"]["axis_correction"] = dataset_infos.axis_correction
             tasks.append("reconstruction")
@@ -348,17 +362,18 @@ class ProcessConfig:
             options["reconstruction"] = copy_dict_items(
                 nabu_config["reconstruction"],
                 ["method", "rotation_axis_position", "fbp_filter_type",
-                "padding_type", "enable_halftomo",
-                "start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
+                "padding_type", "start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
             )
             rec_options = options["reconstruction"]
             rec_options["rotation_axis_position"] = dataset_infos.axis_position
+            rec_options["enable_halftomo"] = self.do_halftomo
             options["build_sino"]["rotation_axis_position"] = dataset_infos.axis_position
+            options["build_sino"]["enable_halftomo"] = self.do_halftomo
             rec_options["axis_correction"] = dataset_infos.axis_correction
             rec_options["angles"] = dataset_infos.reconstruction_angles
             rec_options["radio_dims_y_x"] = dataset_infos.radio_dims[::-1]
             rec_options["pixel_size_cm"] = dataset_infos.pixel_size * 1e-4 # pix size is in microns
-            if rec_options["enable_halftomo"]:
+            if self.do_halftomo:
                 rec_options["angles"] = rec_options["angles"][:rec_options["angles"].size//2]
                 cor_i = int(round(rec_options["rotation_axis_position"]))
                 # New keys

nabu/resources/validators.py

@@ -212,6 +212,14 @@ def boolean_validator(val):
     assert error is None, "Invalid boolean value"
     return res
 
+@validator
+def boolean_or_auto_validator(val):
+    res, error = convert_to_bool(val)
+    if error is not None:
+        assert val.lower() == "auto", "Valid values are 0, 1 and auto"
+        return val
+    return res
+
 @validator
 def float_validator(val):
     val_float, error = convert_to_float(val)
@@ -333,6 +341,14 @@ def phase_method_validator(val):
         replacements=phase_retrieval_methods
     )
 
+@validator
+def unsharp_method_validator(val):
+    return name_range_checker(
+        val,
+        set(unsharp_methods.values()),
+        "unsharp mask method",
+        replacements=phase_retrieval_methods
+    )
 
 @validator
 def padding_mode_validator(val):

setup.py

@@ -57,7 +57,8 @@ def setup_package():
             'psutil',
             'pytest',
             'numpy > 1.9.0',
-            'silx >= 0.14.0',
+            'scipy',
+            'silx >= 0.15.0',
             'distributed',
             'dask_jobqueue',
             'tomoscan >= 0.4.0',