Commit 52f2fecf authored by Pierre Paleo's avatar Pierre Paleo
Browse files

Merge branch 'misc_2021.1.0' into 'master'

Miscellaneous fixes/improvements

Closes #245, #138, #231, #191, and #202

See merge request !134
parents be870363 8363ce5b
Pipeline #47013 failed with stages
in 9 minutes and 44 seconds
# Change Log # Change Log
## 2021.1.1
### Application
- Added `unsharp_method` to select unsharp mask method (gaussian or log)
- `enable_halftomo` can now be set to `auto` to let nabu determine if half aqcuisition was used
### Library
- `scipy` is now a mandatory dependency
## 2021.1.0 ## 2021.1.0
### Application ### Application
......
...@@ -98,10 +98,10 @@ API: [MunchDeringer](apidoc/nabu.preproc.rings) and [CudaMunchDeringer](apidoc/n ...@@ -98,10 +98,10 @@ API: [MunchDeringer](apidoc/nabu.preproc.rings) and [CudaMunchDeringer](apidoc/n
### Flats distortion correction ### Flats distortion correction
A flats distortion estimation/correction is available. If activated, each radio is correlated with its corresponding flat, in order to determine and correct the flat distortion. A flats distortion estimation/correction is available. If activated, each radio is correlated with its corresponding flat, in order to determine and correct the flat distortion.
```{warning} ```{warning}
This is currently quite slow (many correlations), and only supported by the "full radios pipeline" This is currently quite slow (many correlations), and only supported by the "full radios pipeline"
``` ```
Configuration file: `[preproc]`: `flat_distortion_params` and `flat_distortion_correction_enabled` Configuration file: `[preproc]`: `flat_distortion_params` and `flat_distortion_correction_enabled`
...@@ -145,7 +145,7 @@ Although it is a general-purpose image processing utility, unsharp mask is often ...@@ -145,7 +145,7 @@ Although it is a general-purpose image processing utility, unsharp mask is often
Each radio is processed as `UnsharpedImage = (1 + coeff)*Image - coeff * ConvolvedImage` where `ConvolvedImage` is the result of a Gaussian blur applied on the image. Each radio is processed as `UnsharpedImage = (1 + coeff)*Image - coeff * ConvolvedImage` where `ConvolvedImage` is the result of a Gaussian blur applied on the image.
Configuration file: section `[phase]`: `unsharp_coeff = 1.` and `unsharp_sigma = 1.` Configuration file: section `[phase]`: `unsharp_coeff = 1.`, `unsharp_sigma = 1.`, `unsharp_method = gaussian`.
Setting `coeff` to zero (default) disables unsharp masking. Setting `coeff` to zero (default) disables unsharp masking.
......
...@@ -559,7 +559,7 @@ class FullFieldPipeline: ...@@ -559,7 +559,7 @@ class FullFieldPipeline:
self.unsharp_mask = self.UnsharpMaskClass( self.unsharp_mask = self.UnsharpMaskClass(
self._get_shape("unsharp_mask"), self._get_shape("unsharp_mask"),
options["unsharp_sigma"], options["unsharp_coeff"], options["unsharp_sigma"], options["unsharp_coeff"],
mode="reflect", method="gaussian" mode="reflect", method=options["unsharp_method"]
) )
self.register_callback("unsharp_mask", FullFieldPipeline._reshape_radios_after_phase) self.register_callback("unsharp_mask", FullFieldPipeline._reshape_radios_after_phase)
......
...@@ -153,6 +153,7 @@ def validate_nabu_config(config): ...@@ -153,6 +153,7 @@ def validate_nabu_config(config):
if isinstance(config, str): if isinstance(config, str):
config = NabuConfigParser(config).conf_dict config = NabuConfigParser(config).conf_dict
res_config = {} res_config = {}
to_remove = []
for section, section_content in config.items(): for section, section_content in config.items():
# Ignore the "other" section # Ignore the "other" section
if section.lower() == "other": if section.lower() == "other":
...@@ -168,20 +169,27 @@ def validate_nabu_config(config): ...@@ -168,20 +169,27 @@ def validate_nabu_config(config):
continue # deleted key continue # deleted key
if opt is None: if opt is None:
raise ValueError("Unknown option '%s' in section [%s]" % (key, section_updated)) raise ValueError("Unknown option '%s' in section [%s]" % (key, section_updated))
if nabu_config[section_updated][key]["type"] == "unsupported":
to_remove.append((section_updated, key))
continue
validator = nabu_config[section_updated][key]["validator"] validator = nabu_config[section_updated][key]["validator"]
if section_updated not in res_config: # missing section - handled later if section_updated not in res_config: # missing section - handled later
continue continue
res_config[section_updated][key] = validator(section_updated, key, value) res_config[section_updated][key] = validator(section_updated, key, value)
# Remove 'unsupported' params
for param in to_remove:
res_config[param[0]].pop(param[1])
# Handle sections missing in config # Handle sections missing in config
for section in (set(nabu_config.keys()) - set(res_config.keys())): for section in (set(nabu_config.keys()) - set(res_config.keys())):
res_config[section] = _extract_nabuconfig_section(section) res_config[section] = _extract_nabuconfig_section(section)
for key, value in res_config[section].items(): for key, value in res_config[section].items():
if nabu_config[section][key]["type"] == "unsupported":
continue
validator = nabu_config[section][key]["validator"] validator = nabu_config[section][key]["validator"]
res_config[section][key] = validator(section, key, value) res_config[section][key] = validator(section, key, value)
return res_config return res_config
def convert_dict_values(dic, val_replacements, bytes_tostring=False): def convert_dict_values(dic, val_replacements, bytes_tostring=False):
""" """
Modify a dictionary to be able to export it with silx.io.dicttoh5 Modify a dictionary to be able to export it with silx.io.dicttoh5
...@@ -220,6 +228,10 @@ def export_dict_to_h5(dic, h5file, h5path, overwrite_data=True, mode="a"): ...@@ -220,6 +228,10 @@ def export_dict_to_h5(dic, h5file, h5path, overwrite_data=True, mode="a"):
mode: str, optional mode: str, optional
File mode. Default is "a" (append). File mode. Default is "a" (append).
""" """
# in silx >= 0.15, overwrite_data = True becomes update_mode='modify
# and overwrite_data=False becomes update_mode='add'
update_mode = ["add", "modify"][overwrite_data]
#
modified_dic = convert_dict_values( modified_dic = convert_dict_values(
dic, dic,
{None: "None"}, {None: "None"},
...@@ -228,7 +240,7 @@ def export_dict_to_h5(dic, h5file, h5path, overwrite_data=True, mode="a"): ...@@ -228,7 +240,7 @@ def export_dict_to_h5(dic, h5file, h5path, overwrite_data=True, mode="a"):
modified_dic, modified_dic,
h5file=h5file, h5file=h5file,
h5path=h5path, h5path=h5path,
overwrite_data=overwrite_data, update_mode=update_mode,
mode=mode mode=mode
) )
......
...@@ -196,6 +196,12 @@ nabu_config = { ...@@ -196,6 +196,12 @@ nabu_config = {
"validator": float_validator, "validator": float_validator,
"type": "optional", "type": "optional",
}, },
"unsharp_method": {
"default": "gaussian",
"help": "Which type of unsharp mask filter to use. Available values are gaussian (UnsharpedImage = (1 + coeff)*originalPaganinImage - coeff * ConvolvedImage) and laplacian (UnsharpedImage = originalPaganinImage + coeff * ConvolvedImage). Default is gaussian.",
"validator": unsharp_method_validator,
"type": "optional",
},
"padding_type": { "padding_type": {
"default": "edge", "default": "edge",
"help": "Padding type for the filtering step in Paganin/CTF. Available are: mirror, edge, zeros", "help": "Padding type for the filtering step in Paganin/CTF. Available are: mirror, edge, zeros",
...@@ -283,9 +289,9 @@ nabu_config = { ...@@ -283,9 +289,9 @@ nabu_config = {
"type": "optional", # put "advanced" with default value "edges" ? "type": "optional", # put "advanced" with default value "edges" ?
}, },
"enable_halftomo": { "enable_halftomo": {
"default": "0", "default": "auto",
"help": "Whether to enable half-acquisition", "help": "Whether to enable half-acquisition. Default is auto. You can enable/disable it manually by setting 1 or 0.",
"validator": boolean_validator, "validator": boolean_or_auto_validator,
"type": "optional", "type": "optional",
}, },
"start_x": { "start_x": {
...@@ -406,13 +412,13 @@ nabu_config = { ...@@ -406,13 +412,13 @@ nabu_config = {
"default": "1", "default": "1",
"help": "Number of GPUs to use.", "help": "Number of GPUs to use.",
"validator": nonnegative_integer_validator, "validator": nonnegative_integer_validator,
"type": "unsupported", "type": "advanced",
}, },
"gpu_id": { "gpu_id": {
"default": "", "default": "",
"help": "For method = local only. List of GPU IDs to use. This parameter overwrites 'gpus'.\nIf left blank, exactly one GPU will be used, and the best one will be picked.", "help": "For method = local only. List of GPU IDs to use. This parameter overwrites 'gpus'.\nIf left blank, exactly one GPU will be used, and the best one will be picked.",
"validator": list_of_int_validator, "validator": list_of_int_validator,
"type": "unsupported", "type": "advanced",
}, },
"cpu_workers": { "cpu_workers": {
"default": "0", "default": "0",
......
...@@ -15,6 +15,14 @@ phase_retrieval_methods = { ...@@ -15,6 +15,14 @@ phase_retrieval_methods = {
"ctf": "CTF", "ctf": "CTF",
} }
unsharp_methods = {
"gaussian": "gaussian",
"log": "log",
"laplacian": "log",
"none": None,
"": None,
}
padding_modes = { padding_modes = {
"edges": "edge", "edges": "edge",
"edge": "edge", "edge": "edge",
......
...@@ -115,27 +115,28 @@ class ProcessConfig: ...@@ -115,27 +115,28 @@ class ProcessConfig:
def _get_cor(self): def _get_cor(self):
cor = self.nabu_config["reconstruction"]["rotation_axis_position"] rec_config = self.nabu_config["reconstruction"]
cor = rec_config["rotation_axis_position"]
if isinstance(cor, str): # auto-CoR if isinstance(cor, str): # auto-CoR
cor_slice = self.nabu_config["reconstruction"]["cor_slice"] cor_slice = rec_config["cor_slice"]
if cor_slice is not None or cor == "sino-coarse-to-fine": if cor_slice is not None or cor == "sino-coarse-to-fine":
subsampling = extract_parameters( subsampling = extract_parameters(
self.nabu_config["reconstruction"]["cor_options"] rec_config["cor_options"]
).get("subsampling", 10) ).get("subsampling", 10)
self.corfinder = SinoCOREstimator( self.corfinder = SinoCOREstimator(
self.dataset_infos, self.dataset_infos,
cor_slice or 0, cor_slice or 0,
subsampling=subsampling, subsampling=subsampling,
do_flatfield=self.nabu_config["preproc"]["flatfield_enabled"], do_flatfield=self.nabu_config["preproc"]["flatfield_enabled"],
cor_options=self.nabu_config["reconstruction"]["cor_options"], cor_options=rec_config["cor_options"],
logger=self.logger logger=self.logger
) )
else: else:
self.corfinder = COREstimator( self.corfinder = COREstimator(
self.dataset_infos, self.dataset_infos,
halftomo=self.nabu_config["reconstruction"]["enable_halftomo"], halftomo=self.do_halftomo,
do_flatfield=self.nabu_config["preproc"]["flatfield_enabled"], do_flatfield=self.nabu_config["preproc"]["flatfield_enabled"],
cor_options=self.nabu_config["reconstruction"]["cor_options"], cor_options=rec_config["cor_options"],
logger=self.logger logger=self.logger
) )
cor = self.corfinder.find_cor(method=cor) cor = self.corfinder.find_cor(method=cor)
...@@ -162,6 +163,20 @@ class ProcessConfig: ...@@ -162,6 +163,20 @@ class ProcessConfig:
self.dataset_infos.detector_tilt = tilt self.dataset_infos.detector_tilt = tilt
@property
def do_halftomo(self):
"""
Return True if the current dataset is to be reconstructed using 'half-acquisition' setting.
"""
enable_halftomo = self.nabu_config["reconstruction"]["enable_halftomo"]
is_halftomo_dataset = self.dataset_infos.is_halftomo
if enable_halftomo == "auto":
if is_halftomo_dataset is None:
raise ValueError("enable_halftomo was set to 'auto', but information on field of view was not found. Please set either 0 or 1 for enable_halftomo")
return is_halftomo_dataset
return enable_halftomo
def validation_stage2(self): def validation_stage2(self):
validator = NabuValidator(self.nabu_config, self.dataset_infos) validator = NabuValidator(self.nabu_config, self.dataset_infos)
if self.checks: if self.checks:
...@@ -182,7 +197,7 @@ class ProcessConfig: ...@@ -182,7 +197,7 @@ class ProcessConfig:
phase_method = self.nabu_config["phase"]["method"] phase_method = self.nabu_config["phase"]["method"]
do_ctf = phase_method == "CTF" do_ctf = phase_method == "CTF"
do_pag = phase_method == "paganin" do_pag = phase_method == "paganin"
do_unsharp = self.nabu_config["phase"]["unsharp_coeff"] > 0 do_unsharp = self.nabu_config["phase"]["unsharp_method"] is not None and self.nabu_config["phase"]["unsharp_coeff"] > 0
if user_rotate_projections is None and tilt is None: if user_rotate_projections is None and tilt is None:
return None return None
if do_ctf: if do_ctf:
...@@ -286,10 +301,10 @@ class ProcessConfig: ...@@ -286,10 +301,10 @@ class ProcessConfig:
# #
# Unsharp # Unsharp
# #
if nabu_config["phase"]["unsharp_coeff"] > 0: if nabu_config["phase"]["unsharp_method"] is not None and nabu_config["phase"]["unsharp_coeff"] > 0:
tasks.append("unsharp_mask") tasks.append("unsharp_mask")
options["unsharp_mask"] = copy_dict_items( options["unsharp_mask"] = copy_dict_items(
nabu_config["phase"], ["unsharp_coeff", "unsharp_sigma"] nabu_config["phase"], ["unsharp_coeff", "unsharp_sigma", "unsharp_method"]
) )
# #
# -logarithm # -logarithm
...@@ -339,8 +354,7 @@ class ProcessConfig: ...@@ -339,8 +354,7 @@ class ProcessConfig:
tasks.append("build_sino") tasks.append("build_sino")
options["build_sino"] = copy_dict_items( options["build_sino"] = copy_dict_items(
nabu_config["reconstruction"], nabu_config["reconstruction"],
["rotation_axis_position", "enable_halftomo", "start_x", "end_x", ["rotation_axis_position", "start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
"start_y", "end_y", "start_z", "end_z"]
) )
options["build_sino"]["axis_correction"] = dataset_infos.axis_correction options["build_sino"]["axis_correction"] = dataset_infos.axis_correction
tasks.append("reconstruction") tasks.append("reconstruction")
...@@ -348,17 +362,18 @@ class ProcessConfig: ...@@ -348,17 +362,18 @@ class ProcessConfig:
options["reconstruction"] = copy_dict_items( options["reconstruction"] = copy_dict_items(
nabu_config["reconstruction"], nabu_config["reconstruction"],
["method", "rotation_axis_position", "fbp_filter_type", ["method", "rotation_axis_position", "fbp_filter_type",
"padding_type", "enable_halftomo", "padding_type", "start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
"start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
) )
rec_options = options["reconstruction"] rec_options = options["reconstruction"]
rec_options["rotation_axis_position"] = dataset_infos.axis_position rec_options["rotation_axis_position"] = dataset_infos.axis_position
rec_options["enable_halftomo"] = self.do_halftomo
options["build_sino"]["rotation_axis_position"] = dataset_infos.axis_position options["build_sino"]["rotation_axis_position"] = dataset_infos.axis_position
options["build_sino"]["enable_halftomo"] = self.do_halftomo
rec_options["axis_correction"] = dataset_infos.axis_correction rec_options["axis_correction"] = dataset_infos.axis_correction
rec_options["angles"] = dataset_infos.reconstruction_angles rec_options["angles"] = dataset_infos.reconstruction_angles
rec_options["radio_dims_y_x"] = dataset_infos.radio_dims[::-1] rec_options["radio_dims_y_x"] = dataset_infos.radio_dims[::-1]
rec_options["pixel_size_cm"] = dataset_infos.pixel_size * 1e-4 # pix size is in microns rec_options["pixel_size_cm"] = dataset_infos.pixel_size * 1e-4 # pix size is in microns
if rec_options["enable_halftomo"]: if self.do_halftomo:
rec_options["angles"] = rec_options["angles"][:rec_options["angles"].size//2] rec_options["angles"] = rec_options["angles"][:rec_options["angles"].size//2]
cor_i = int(round(rec_options["rotation_axis_position"])) cor_i = int(round(rec_options["rotation_axis_position"]))
# New keys # New keys
......
...@@ -212,6 +212,14 @@ def boolean_validator(val): ...@@ -212,6 +212,14 @@ def boolean_validator(val):
assert error is None, "Invalid boolean value" assert error is None, "Invalid boolean value"
return res return res
@validator
def boolean_or_auto_validator(val):
res, error = convert_to_bool(val)
if error is not None:
assert val.lower() == "auto", "Valid values are 0, 1 and auto"
return val
return res
@validator @validator
def float_validator(val): def float_validator(val):
val_float, error = convert_to_float(val) val_float, error = convert_to_float(val)
...@@ -333,6 +341,14 @@ def phase_method_validator(val): ...@@ -333,6 +341,14 @@ def phase_method_validator(val):
replacements=phase_retrieval_methods replacements=phase_retrieval_methods
) )
@validator
def unsharp_method_validator(val):
return name_range_checker(
val,
set(unsharp_methods.values()),
"unsharp mask method",
replacements=phase_retrieval_methods
)
@validator @validator
def padding_mode_validator(val): def padding_mode_validator(val):
......
...@@ -57,7 +57,8 @@ def setup_package(): ...@@ -57,7 +57,8 @@ def setup_package():
'psutil', 'psutil',
'pytest', 'pytest',
'numpy > 1.9.0', 'numpy > 1.9.0',
'silx >= 0.14.0', 'scipy',
'silx >= 0.15.0',
'distributed', 'distributed',
'dask_jobqueue', 'dask_jobqueue',
'tomoscan >= 0.4.0', 'tomoscan >= 0.4.0',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment