Skip to content
Snippets Groups Projects
Commit 2f2008b9 authored by casagran's avatar casagran
Browse files

get rid of zwise median and create option for zwise mean

parent d6954282
No related branches found
No related tags found
1 merge request!6Draft: Resolve "py-bkg-rm"
Hide whitespace changes
Inline Side-by-side
Showing
with 187 additions and 75 deletions
zUtil_Python/pydct/orangecontrib/pydct/pipelines/.preprocess-rnice.ows.swp.p deleted 100644 → 0
+ 0
− 0
View file @ d6954282
File deleted
zUtil_Python/pydct/orangecontrib/pydct/pipelines/preprocess-joaopcbertoldo.ows
+ 45
− 16
View file @ 2f2008b9
<?xml version='1.0' encoding='utf-8'?>
<scheme version="2.0" title="" description="">
<nodes>
<node id="0" name="Load _data from URL" qualified_name="orangecontrib.pydct.widgets.load_data.LoadDataFromDataUrl" project_name="pydct" version="" title="load data" position="(93.0, 437.0)" />
<node id="1" name="Load _data from URL" qualified_name="orangecontrib.pydct.widgets.load_data.LoadDataFromDataUrl" project_name="pydct" version="" title="load dark" position="(88.0, 335.0)" />
<node id="2" name="z-wise median" qualified_name="orangecontrib.pydct.widgets.zwise_median.ZwiseMedian" project_name="pydct" version="" title="Z-wise median" position="(344.0, 331.0)" />
<node id="3" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="roi selection" position="(235.0, 220.0)" />
<node id="4" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="roi selection (1)" position="(520.0, 202.0)" />
<node id="5" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="margin selection" position="(238.0, 515.0)" />
<node id="6" name="subtract dark" qualified_name="orangecontrib.pydct.widgets.subtract_dark.SubtractDark" project_name="pydct" version="" title="Subtract dark" position="(585.0, 414.0)" />
<node id="7" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="roi selection (2)" position="(718.0, 270.0)" />
<node id="0" name="Load _data from URL" qualified_name="orangecontrib.pydct.widgets.load_data.LoadDataFromDataUrl" project_name="pydct" version="" title="load data" position="(283.0, 180.0)" />
<node id="1" name="Load _data from URL" qualified_name="orangecontrib.pydct.widgets.load_data.LoadDataFromDataUrl" project_name="pydct" version="" title="load dark" position="(138.0, 307.0)" />
<node id="2" name="z-wise median" qualified_name="orangecontrib.pydct.widgets.zwise_median.ZwiseMedian" project_name="pydct" version="" title="Z-wise median" position="(310.0, 266.0)" />
<node id="3" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz dark raw" position="(255.0, 425.0)" />
<node id="4" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz dark reduced" position="(503.0, 427.0)" />
<node id="5" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="margin selection" position="(696.0, 385.0)" />
<node id="6" name="subtract dark" qualified_name="orangecontrib.pydct.widgets.subtract_dark.SubtractDark" project_name="pydct" version="" title="Subtract dark" position="(543.0, 249.0)" />
<node id="7" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz data - dark" position="(686.0, 96.0)" />
<node id="8" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz data raw" position="(417.0, 54.0)" />
<node id="9" name="normalize margin mean" qualified_name="orangecontrib.pydct.widgets.normalize_margin_mean.NormalizeMargin" project_name="pydct" version="" title="normalize margin mean" position="(964.0, 261.0)" />
<node id="10" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz data normalized" position="(1117.0, 102.0)" />
<node id="11" name="allocate shared memory" qualified_name="orangecontrib.pydct.widgets.shared_memory_ndarray.AllocateSharedMemory" project_name="pydct" version="" title="allocate shared memory" position="(1101.0, 271.0)" />
<node id="12" name="remove z-moving medians" qualified_name="orangecontrib.pydct.widgets.remove_zmoving_medians.RemoveZmovingMedian" project_name="pydct" version="" title="remove z-moving medians" position="(1281.0, 269.0)" />
<node id="13" name="xy-filter median" qualified_name="orangecontrib.pydct.widgets.xyfilter_median.XYFilterMedian" project_name="pydct" version="" title="xy-filter median" position="(1460.0, 318.0)" />
<node id="14" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz bkg" position="(1414.0, -8.0)" />
<node id="15" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="viz data - bkg" position="(1454.0, 108.0)" />
<node id="16" name="roi selection" qualified_name="orangecontrib.pydct.widgets.roi_selection.RoiSelectionWidgetOW" project_name="pydct" version="" title="roi selection" position="(1605.0, 177.0)" />
</nodes>
<links>
<link id="0" source_node_id="0" sink_node_id="5" source_channel="Data" sink_channel="data" enabled="true" />
<link id="1" source_node_id="0" sink_node_id="6" source_channel="Data" sink_channel="data" enabled="true" />
<link id="2" source_node_id="1" sink_node_id="3" source_channel="Data" sink_channel="data" enabled="true" />
<link id="3" source_node_id="1" sink_node_id="2" source_channel="Data" sink_channel="data" enabled="true" />
<link id="4" source_node_id="2" sink_node_id="4" source_channel="data_squeezed" sink_channel="data" enabled="true" />
<link id="5" source_node_id="2" sink_node_id="6" source_channel="data_squeezed" sink_channel="dark" enabled="true" />
<link id="6" source_node_id="6" sink_node_id="7" source_channel="data" sink_channel="data" enabled="true" />
<link id="0" source_node_id="0" sink_node_id="6" source_channel="Data" sink_channel="data" enabled="true" />
<link id="1" source_node_id="1" sink_node_id="3" source_channel="Data" sink_channel="data" enabled="true" />
<link id="2" source_node_id="1" sink_node_id="2" source_channel="Data" sink_channel="data" enabled="true" />
<link id="3" source_node_id="2" sink_node_id="4" source_channel="data_squeezed" sink_channel="data" enabled="true" />
<link id="4" source_node_id="2" sink_node_id="6" source_channel="data_squeezed" sink_channel="dark" enabled="true" />
<link id="5" source_node_id="6" sink_node_id="7" source_channel="data" sink_channel="data" enabled="true" />
<link id="6" source_node_id="6" sink_node_id="5" source_channel="data" sink_channel="data" enabled="true" />
<link id="7" source_node_id="0" sink_node_id="8" source_channel="Data" sink_channel="data" enabled="true" />
<link id="8" source_node_id="5" sink_node_id="9" source_channel="roi_origin" sink_channel="margin_roi_origin" enabled="true" />
<link id="9" source_node_id="5" sink_node_id="9" source_channel="roi_size" sink_channel="margin_roi_size" enabled="true" />
<link id="10" source_node_id="6" sink_node_id="9" source_channel="data" sink_channel="data" enabled="true" />
<link id="11" source_node_id="9" sink_node_id="10" source_channel="data" sink_channel="data" enabled="true" />
<link id="12" source_node_id="9" sink_node_id="11" source_channel="data" sink_channel="data" enabled="true" />
<link id="13" source_node_id="11" sink_node_id="12" source_channel="data_shared" sink_channel="shared_data" enabled="true" />
<link id="14" source_node_id="12" sink_node_id="13" source_channel="data_shared" sink_channel="shared_data" enabled="true" />
<link id="15" source_node_id="12" sink_node_id="14" source_channel="backgrounds" sink_channel="data" enabled="true" />
<link id="16" source_node_id="12" sink_node_id="15" source_channel="data" sink_channel="data" enabled="true" />
<link id="17" source_node_id="13" sink_node_id="16" source_channel="data" sink_channel="data" enabled="true" />
</links>
<annotations />
<thumbnail />
<node_properties>
<properties node_id="0" format="literal">{'controlAreaVisible': True, 'data_link': '/entry_0000/ESRF-ID11/marana/data', 'file_path': '/home/joaopcbertoldo/marana_0000.h5', 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x08\x8e\x00\x00\x03%\x00\x00\n\xc3\x00\x00\x04H\x00\x00\x08\x8e\x00\x00\x03%\x00\x00\n\xc3\x00\x00\x04H\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\x08\x8e\x00\x00\x03%\x00\x00\n\xc3\x00\x00\x04H', '__version__': 1}</properties>
<properties node_id="1" format="literal">{'controlAreaVisible': True, 'data_link': '/entry_0000/ESRF-ID11/marana/data', 'file_path': '/home/joaopcbertoldo/dark.h5', 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x08\x8e\x00\x00\x03%\x00\x00\n\xc3\x00\x00\x04H\x00\x00\x08\x8e\x00\x00\x03%\x00\x00\n\xc3\x00\x00\x04H\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\x08\x8e\x00\x00\x03%\x00\x00\n\xc3\x00\x00\x04H', '__version__': 1}</properties>
<properties node_id="1" format="literal">{'controlAreaVisible': True, 'data_link': '/entry_0000/ESRF-ID11/marana/data', 'file_path': '/home/joaopcbertoldo/dark.h5', 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x03 \x00\x00\x02\xcb\x00\x00\x05U\x00\x00\x04\x13\x00\x00\x03 \x00\x00\x02\xf0\x00\x00\x05U\x00\x00\x04\x13\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\x03 \x00\x00\x02\xf0\x00\x00\x05U\x00\x00\x04\x13', '__version__': 1}</properties>
<properties node_id="2" format="literal">{'controlAreaVisible': True, 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\t\x90\x00\x00\x03H\x00\x00\n\x90\x00\x00\x03\xea\x00\x00\t\x90\x00\x00\x03m\x00\x00\n\x90\x00\x00\x03\xea\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\t\x90\x00\x00\x03m\x00\x00\n\x90\x00\x00\x03\xea', '__version__': 1}</properties>
<properties node_id="3" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b"\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x08\x1a\x00\x00\x02I\x00\x00\x0c'\x00\x00\x05\t\x00\x00\x08\x1a\x00\x00\x02n\x00\x00\x0c'\x00\x00\x05\t\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\x08\x1a\x00\x00\x02n\x00\x00\x0c'\x00\x00\x05\t", '__version__': 1}</properties>
<properties node_id="4" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b"\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x08\x1a\x00\x00\x02I\x00\x00\x0c'\x00\x00\x05\t\x00\x00\x08\x1a\x00\x00\x02n\x00\x00\x0c'\x00\x00\x05\t\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\x08\x1a\x00\x00\x02n\x00\x00\x0c'\x00\x00\x05\t", '__version__': 1}</properties>
<properties node_id="5" format="literal">{'controlAreaVisible': True, 'roi_origin': [726.0846254927728, 890.9977011494249], 'roi_size': [46.647393780113816, 285.42528735632175], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\x7f\x00\x00\x04\xaf\x00\x00\x00\t\x00\x00\x00\x14\x00\x00\x07\x88\x00\x00\x04\x9e\x00\x00\x00\x01\x02\x00\x00\x00\x07\x80\x00\x00\x00\x00\x00\x00\x00%\x00\x00\x07\x7f\x00\x00\x04\xaf', '__version__': 1}</properties>
<properties node_id="6" format="literal">{'controlAreaVisible': True, 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\t\xbd\x00\x00\x02\x8c\x00\x00\n\x84\x00\x00\x04\x7f\x00\x00\t\xbd\x00\x00\x02\x8c\x00\x00\n\x84\x00\x00\x04\x7f\x00\x00\x00\x00\x00\x00\x00\x00\x05V\x00\x00\t\xbd\x00\x00\x02\x8c\x00\x00\n\x84\x00\x00\x04\x7f', '__version__': 1}</properties>
<properties node_id="7" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\x7f\x00\x00\x04\xaf\x00\x00\x03r\x00\x00\x00%\x00\x00\x07\x7f\x00\x00\x02\xc0\x00\x00\x00\x01\x02\x00\x00\x00\x07\x80\x00\x00\x00\x00\x00\x00\x00%\x00\x00\x07\x7f\x00\x00\x04\xaf', '__version__': 1}</properties>
<properties node_id="8" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x01\\\x00\x00\x00\xe8\x00\x00\x04\xac\x00\x00\x03\xc1\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd', '__version__': 1}</properties>
<properties node_id="9" format="literal">{'controlAreaVisible': True, 'savedWidgetGeometry': None, '__version__': 1}</properties>
<properties node_id="10" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x01\\\x00\x00\x00\xe8\x00\x00\x04\xac\x00\x00\x03\xc1\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd', '__version__': 1}</properties>
<properties node_id="11" format="literal">{'controlAreaVisible': True, 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x04\x85\x00\x00\x01R\x00\x00\x06\x13\x00\x00\x02\xcf\x00\x00\x04\x86\x00\x00\x01h\x00\x00\x06\x12\x00\x00\x02\xcb\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x04\x86\x00\x00\x01h\x00\x00\x06\x12\x00\x00\x02\xcb', '__version__': 1}</properties>
<properties node_id="12" format="literal">{'controlAreaVisible': True, 'median_validity': 50, 'median_window': 500, 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x03Z\x00\x00\x01)\x00\x00\x04#\x00\x00\x036\x00\x00\x03[\x00\x00\x01?\x00\x00\x04"\x00\x00\x032\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x03[\x00\x00\x01?\x00\x00\x04"\x00\x00\x032', '__version__': 1}</properties>
<properties node_id="13" format="literal">{'controlAreaVisible': True, 'kernel_size_x': 3, 'kernel_size_y': 3, 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x03Z\x00\x00\x01)\x00\x00\x04#\x00\x00\x036\x00\x00\x03[\x00\x00\x01?\x00\x00\x04"\x00\x00\x032\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x03[\x00\x00\x01?\x00\x00\x04"\x00\x00\x032', '__version__': 1}</properties>
<properties node_id="14" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x01\\\x00\x00\x00\xe8\x00\x00\x04\xac\x00\x00\x03\xc1\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd', '__version__': 1}</properties>
<properties node_id="15" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x01\\\x00\x00\x00\xe8\x00\x00\x04\xac\x00\x00\x03\xc1\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x01]\x00\x00\x00\xfe\x00\x00\x04\xab\x00\x00\x03\xbd', '__version__': 1}</properties>
<properties node_id="16" format="literal">{'controlAreaVisible': True, 'roi_origin': [], 'roi_size': [], 'savedWidgetGeometry': b'\x01\xd9\xd0\xcb\x00\x03\x00\x00\x00\x00\x00\xbd\x00\x00\x00\x9f\x00\x00\x05g\x00\x00\x03\xae\x00\x00\x00\xbe\x00\x00\x00\xb5\x00\x00\x05f\x00\x00\x03\xaa\x00\x00\x00\x00\x00\x00\x00\x00\x07\x80\x00\x00\x00\xbe\x00\x00\x00\xb5\x00\x00\x05f\x00\x00\x03\xaa', '__version__': 1}</properties>
</node_properties>
<session_state>
<window_groups />
......
zUtil_Python/pydct/orangecontrib/pydct/widgets/load_data.py
+ 7
− 7
View file @ 2f2008b9
"""
Open 3D _data from h5 files and return numpy array.
Open 3D data from h5 files and return numpy array.
"""
# todo make it terminate if stuff change while processing
......@@ -21,7 +21,7 @@ _logger = logging.getLogger("orangecontrib.pydct")
class LoadDataThread(QThread):
"""load the _data using silx's `get_data`"""
"""load the data using silx's `get_data`"""
def __init__(self, parent, url):
super().__init__(parent=parent)
......@@ -29,7 +29,7 @@ class LoadDataThread(QThread):
self.data = None
def run(self):
"""load the _data using silx's `get_data`"""
"""load the data using silx's `get_data`"""
try:
data = common.catch_silx_io_exceptions(get_data)(self.url)
......@@ -40,14 +40,14 @@ class LoadDataThread(QThread):
except Bad3DDataError as ex:
_logger.exception(ex)
# _logger.warning("_data is not a 3D dataset?")
# _logger.warning("data is not a 3D dataset?")
else:
self.data = data
class LoadDataFromDataUrl(OWWidget):
"""Build a silx.io.url.DataUrl and load the 3D _data pointed by it."""
"""Build a silx.io.url.DataUrl and load the 3D data pointed by it."""
name = "Load _data from URL"
description = "Load a 3D dataset from an h5 url (scheme + file path + group link) as in silx.io.url"
......@@ -86,7 +86,7 @@ class LoadDataFromDataUrl(OWWidget):
callback=self._select_filename,
)
# ========================= _data link =========================
# ========================= data link =========================
data_link_box = gui.widgetBox(self.controlArea, "data link")
......@@ -157,7 +157,7 @@ class LoadDataFromDataUrl(OWWidget):
self._url_label.setText(self.url_display)
# if self.url is None:
# self.error("Pick an h5 file and filling the _data path!")
# self.error("Pick an h5 file and filling the data path!")
# return
dataurl = DataUrl(self.url)
......
zUtil_Python/pydct/pydct/preprocess/__init__.py
+ 1
− 1
View file @ 2f2008b9
......@@ -13,4 +13,4 @@ DCT preprocessing operations:
__authors__ = ["Joao P C Bertoldo"]
__date__ = "2021-05"
from . import moving_median_removal, xyfilter_median, zwise_median
from . import moving_median_removal, xyfilter_median
zUtil_Python/pydct/pydct/preprocess/_fixtures.py
+ 7
− 7
View file @ 2f2008b9
......@@ -13,9 +13,9 @@ from silx.io.url import DataUrl
@pytest.fixture(scope="session")
def realdata00_h5():
"""a volume from marana with 100 (2048, 2048)-frames"""
dir_ = Path("/_data/id11/3dxrd/blc12852/id11/bmg_l1/bmg_l1_bmg_dct2/scan0002")
dir_ = Path("/data/id11/3dxrd/blc12852/id11/bmg_l1/bmg_l1_bmg_dct2/scan0002")
path = dir_ / "marana_0000.h5"
datalink = "/entry_0000/ESRF-ID11/marana/_data"
datalink = "/entry_0000/ESRF-ID11/marana/data"
urlstr = f"silx:{str(path)}::{datalink}"
return path, datalink, urlstr
......@@ -26,7 +26,7 @@ def realdata00_url(realdata00_h5):
path, datalink, urlstr = realdata00_h5
url = DataUrl(urlstr)
# make sure the _data exists
# make sure the data exists
assert path.exists()
assert path.is_file()
assert url.is_valid()
......@@ -39,11 +39,11 @@ def realdata00_url(realdata00_h5):
@pytest.fixture
def output_h5(tmp_path):
"""generic url to use as _data output"""
"""generic url to use as data output"""
path = tmp_path / "output.h5"
grouplink = "/computed_suff"
urlstr = f"silx:{str(path.absolute())}::{grouplink}"
return path, grouplink, urlstr, f"{urlstr}/_data"
return path, grouplink, urlstr, f"{urlstr}/data"
@pytest.fixture
......@@ -66,7 +66,7 @@ def output_url(output_h5):
@pytest.fixture(scope="session")
def darkend00_h5():
"""a volume from marana with 21 (2048, 2048)-frames of a darkend acquisition"""
dir_ = Path("/_data/id11/3dxrd/blc12852/id11/bmg_l1/bmg_l1_bmg_dct2")
dir_ = Path("/data/id11/3dxrd/blc12852/id11/bmg_l1/bmg_l1_bmg_dct2")
path = dir_ / "bmg_l1_bmg_dct2.h5"
datalink = "/10.1/measurement/marana"
urlstr = f"silx:{str(path)}::{datalink}"
......@@ -75,7 +75,7 @@ def darkend00_h5():
@pytest.fixture(scope="session") # session because the `get_data` is slow
def darkend00_url(darkend00_h5):
"""make sure the _data exists (to be used as parameter to the calls)"""
"""make sure the data exists (to be used as parameter to the calls)"""
path, datalink, urlstr = darkend00_h5
url = DataUrl(urlstr)
......
zUtil_Python/pydct/pydct/preprocess/main.py
+ 32
− 12
View file @ 2f2008b9
......@@ -23,7 +23,7 @@ from numpy import ndarray
from pydct import common, log, parse
from pydct.common import get_dataset_info
from pydct.parse import DocstringsActionAbstract, ExampleCallsActionAbstract
from pydct.preprocess import moving_median_removal, xyfilter_median, zwise_median
from pydct.preprocess import xyfilter_median
from pydct.preprocess.moving_median_removal import remove_moving_medians
from pydct.preprocess.xyfilter_median import xyfilter_median
from silx.io import get_data
......@@ -48,12 +48,18 @@ COMMAND_PREPROCESS = "preprocess"
class NormalizationOptions(Enum):
"""Different"""
"""todo doc NormalizationOptions"""
# none = None
margin_mean = "margin_mean"
class DarkZwiseReductionOptions(Enum):
"""todo doc DarkZwiseReductionOptions"""
mean = "mean"
median = "median"
# todo create parser option to show output struct
@dataclasses.dataclass
class Outputs:
......@@ -83,6 +89,7 @@ def main(
nprocs: Optional[int],
update_mode: str,
save_backgrounds: bool,
dark_zwise_reduction: DarkZwiseReductionOptions,
) -> int:
"""
Do the entire preprocessing:
......@@ -144,6 +151,8 @@ def main(
nprocs: nb. of processes used in the parallelized functions; None corresponds to the nb. of cpus
save_backgrounds: if false, the backgrounds are discarded and a `none` is put in its place
dark_zwise_reduction: which reduction operation to apply on each pixel position along the z-axis of the dark image
"""
# ============================ args ============================
......@@ -257,7 +266,16 @@ def main(
logger.info("starting processing")
logger.debug("squeezing the dark volume on the z-axis")
dark = np.squeeze(np.mean(dark, axis=0)) # todo verify if this is median or mean
# todo test both cases
if dark_zwise_reduction == DarkZwiseReductionOptions.mean:
dark = np.squeeze(np.mean(dark, axis=0)) # todo verify if this is median or mean
elif dark_zwise_reduction == DarkZwiseReductionOptions.median:
dark = np.squeeze(np.median(dark, axis=0)) # todo verify if this is median or mean
else:
raise NotImplementedError(f"{dark_zwise_reduction=}")
logger.debug("subtract dark from data")
data = data - dark
......@@ -333,14 +351,7 @@ def normalize_margin_mean(data, margin_bounding_box):
# ======================================== parser ========================================
modules = OrderedDict(
[
(mod.__name__, mod)
for mod in [
zwise_median,
]
]
)
modules = OrderedDict([(mod.__name__, mod) for mod in []])
parser = ArgumentParser(
prog="preprocess",
......@@ -468,6 +479,14 @@ preprocess_parser.add_argument(
dest="filter_dimensions", # must be present so the kwargs shows up in the namespace
)
preprocess_parser.add_argument(
"--dark_zwise_reduction",
choices=list(DarkZwiseReductionOptions),
default=DarkZwiseReductionOptions.mean,
type=DarkZwiseReductionOptions,
help="which reduction operation to apply on each pixel position along the z-axis of the dark image",
)
preprocess_parser.add_argument(
"--nprocs",
help="how many workers should be used in paralel? default is set to the machine's nb of cpus",
......@@ -535,6 +554,7 @@ parser_main = common.namespace2kwargs(
"nprocs",
"update_mode",
"save_backgrounds",
"dark_zwise_reduction",
}
)(main)
......
zUtil_Python/pydct/pydct/preprocess/main_test.py
+ 71
− 8
View file @ 2f2008b9
......@@ -91,7 +91,7 @@ def test_parser_run_with_defaults(parser_file_args, output_h5):
assert path.exists()
assert path.is_file()
data_urlstr = f"{urlstr}/_data" # this is specific to the `Outputs` obj in `main`
data_urlstr = f"{urlstr}/data" # this is specific to the `Outputs` obj in `main`
data = get_data(data_urlstr)
assert data.shape == (100, 2048, 2048)
......@@ -158,18 +158,81 @@ def main_kwargs(realdata00_url, darkend00_url, output_url):
nprocs=None,
update_mode="modify",
save_backgrounds=True,
dark_zwise_reduction=main.DarkZwiseReductionOptions.mean,
)
@pytest.fixture(
params=[
((200, 300), (300, 400)),
((1200, 1300), (1300, 1400)), # todo raise a warning for something so out of the center
]
)
def margin_bounding_box_variations(request):
"""possible, valid margin boxes"""
return request.param
@pytest.fixture(
params=[
(10, 50),
(20, 50),
(20, 20),
]
)
def margin_zwise_moving_median_variations(request):
"""possible, valid margin boxes"""
return dict(
median_validity=request.param[0],
median_window=request.param[1],
)
@pytest.fixture(params=list(main.DarkZwiseReductionOptions))
def dark_zwise_reduction_variations(request):
""""""
return request.param
@pytest.fixture
def main_kwargs_variations(
realdata00_url,
darkend00_url,
output_url,
margin_bounding_box_variations,
margin_zwise_moving_median_variations,
dark_zwise_reduction_variations,
):
"""a possible set of parameters"""
return dict(
input_url=realdata00_url,
dark_url=darkend00_url,
output_url=output_url,
normalization=main.NormalizationOptions.margin_mean,
normalization_numerator=200,
margin_bounding_box=margin_bounding_box_variations,
median_validity=margin_zwise_moving_median_variations["median_validity"],
median_window=margin_zwise_moving_median_variations["median_window"],
filter_dimensions=(3, 3),
nprocs=None,
update_mode="modify",
save_backgrounds=True,
dark_zwise_reduction=dark_zwise_reduction_variations,
)
# ====================================== normal calls
def test_main_dryrun(main_kwargs, monkeypatch):
def test_main_variations_dryrun(main_kwargs_variations, monkeypatch):
median_validity = main_kwargs_variations["median_validity"]
def mock_get_data(*_, **__):
return np.ones((200, 500, 500))
return np.ones((100, 2048, 2048))
def mock_remove_moving_medians(data, *_, **__):
return data, np.ones((20, 500, 500)), None
return data, np.ones((100 // median_validity, 2048, 2048)), None
def mock_xyfilter_median(data, *_, **__):
return data
......@@ -182,7 +245,7 @@ def test_main_dryrun(main_kwargs, monkeypatch):
monkeypatch.setattr(main, "xyfilter_median", mock_xyfilter_median)
monkeypatch.setattr(main, "dicttoh5", mock_dicttoh5)
main.main(**main_kwargs)
main.main(**main_kwargs_variations)
@pytest.mark.slow
......@@ -195,7 +258,7 @@ def test_main(main_kwargs, output_h5):
assert path.exists()
assert path.is_file()
data_urlstr = f"{urlstr}/_data" # this is specific to the `Outputs` obj in `main`
data_urlstr = f"{urlstr}/data" # this is specific to the `Outputs` obj in `main`
data = get_data(data_urlstr)
assert data.shape == (100, 2048, 2048)
......@@ -211,7 +274,7 @@ def test_main(main_kwargs, output_h5):
params=[
lambda s: "," + s, # something wrong
lambda s: s.split(":", 1)[1], # missing scheme
lambda s: s.split("::", 1)[0], # missing _data link
lambda s: s.split("::", 1)[0], # missing data link
]
)
def main_kwargs_invalid_input_url(request, main_kwargs, realdata00_h5):
......@@ -251,7 +314,7 @@ def test_main_kwargs_invalid_moving_window_params(main_kwargs_bad_moving_window_
@pytest.fixture
def main_kwargs_input_doesnt_exist(main_kwargs):
"""should cause file not found"""
input_url = DataUrl("silx:asdf.h5::/_data")
input_url = DataUrl("silx:asdf.h5::/data")
return {**main_kwargs, **dict(input_url=input_url)}
......
zUtil_Python/pydct/pydct/preprocess/moving_median_removal.py
+ 7
− 7
View file @ 2f2008b9
......@@ -80,7 +80,7 @@ def _worker_init(
"""Process one median window in the z-axis for all the xy positions individually.
Args:
mp_data: _data to process (read only)
mp_data: data to process (read only)
mp_medians: where the medians are dumped
data_shape:
data_dtype: [description]
......@@ -131,7 +131,7 @@ def remove_moving_medians(
side of the index range.á
Striding and border effects:
- if `median_validity` is not a factor of the z-axis size of `_data`
- if `median_validity` is not a factor of the z-axis size of `data`
the last window will be valid for less slices and computed will
fewer slices on the righ sice (positive direction of the axis)
- since the `median_window` will usually be bigger than `median_validity`
......@@ -142,18 +142,18 @@ def remove_moving_medians(
(positive direction of the axis)
Args:
data: 3d; the _data volume's axes order is assumed to be (z, x, y)
data: 3d; the data volume's axes order is assumed to be (z, x, y)
median_validity: how many slices is a median valid for?
median_window: how many slices are use to compute a median slice?
nprocs: nb. processes in parallel. Defaults to None (number of cores in the computer).
multiprocessing_rawarray:
if none: will allocate one, create an ndarray from buffer, and copy the content from `_data` to it
if given: `_data` is supposed to be an ndarray from the shared memeory array
if none: will allocate one, create an ndarray from buffer, and copy the content from `data` to it
if given: `data` is supposed to be an ndarray from the shared memeory array
see args in the argparse.parser
Returns:
_data with moving medians removed, medians
data with moving medians removed, medians
if `multiprocessing_rawarray` is None, alsor return the
multiprocessing rawarray object at the last position
"""
......@@ -204,7 +204,7 @@ def remove_moving_medians(
) as pool:
# notice that the median computation and removal cannot be done at once
# because we're modifying _data (shared) directly, so removing the median
# because we're modifying data (shared) directly, so removing the median
# from a slice could make another's computation go wrong
# btw, it's important to do the operation in-place becasue the memory allocation
# takes a considerable time
......
zUtil_Python/pydct/pydct/preprocess/xyfilter_median.py
+ 4
− 4
View file @ 2f2008b9
......@@ -73,11 +73,11 @@ def xyfilter_median(
todo test xyfilter_median
multiprocessing_rawarray:
if none: will allocate one, create an ndarray from buffer, and copy the content from `_data` to it
if given: `_data` is supposed to be an ndarray from the shared memeory array
if none: will allocate one, create an ndarray from buffer, and copy the content from `data` to it
if given: `data` is supposed to be an ndarray from the shared memeory array
Returns:
median-filtered _data
median-filtered data
if `multiprocessing_rawarray` is None, alsor return the
multiprocessing rawarray object at the last position
......@@ -117,7 +117,7 @@ def xyfilter_median(
) as pool:
# notice that the median computation and removal cannot be done at once
# because we're modifying _data (shared) directly, so removing the median
# because we're modifying data (shared) directly, so removing the median
# from a slice could make another's computation go wrong
# btw, it's important to do the operation in-place becasue the memory allocation
# takes a considerable time
......
zUtil_Python/pydct/pydct/preprocess/zwise_median.py → zUtil_Python/pydct/pydct/preprocess/zwise_reduce.py
+ 9
− 9
View file @ 2f2008b9
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Compute the median of all pixels in a volume _data getting a single slice.
This is to be used with the ref and darkend _data.
Compute the median of all pixels in a volume data getting a single slice.
This is to be used with the ref and darkend data.
@author: joaopcbertoldo
"""
......@@ -55,17 +55,17 @@ def main(
update_mode: str,
) -> int:
"""
Compute a z-wise median (one per pixel spatial position) of 3d _data block and save it.
Compute a z-wise median (one per pixel spatial position) of 3d data block and save it.
The median computation is done for each xy-position individually considering all the z-axis values.
!!IMPORTANT!! The _data volume's axes order is assumed to be (z, x, y).
!!IMPORTANT!! The data volume's axes order is assumed to be (z, x, y).
IO: reads _data with `silx.io.get_data` and writes with `silx.io.dictdump.dicttoh5`
IO: reads data with `silx.io.get_data` and writes with `silx.io.dictdump.dicttoh5`
Args:
input_url: url (scheme + path + link) to READ the INPUT _data
file and its internal link with the _data
ex: silx:/_data/id11/my.h5::/2.1/measurement/marana
input_url: url (scheme + path + link) to READ the INPUT data
file and its internal link with the data
ex: silx:/data/id11/my.h5::/2.1/measurement/marana
obs1: 'silx:' indicates the h5 scheme
obs2: silx (the library) also supports 'fabio', but that is not supported here
......@@ -96,7 +96,7 @@ def main(
logger.debug("the args look good to go")
# ============================ _data ============================
# ============================ data ============================
data = common.catch_silx_io_exceptions(get_data, logger)(input_url)
......
zUtil_Python/pydct/pydct/preprocess/zwise_median_test.py → zUtil_Python/pydct/pydct/preprocess/zwise_reduce_test.py
+ 4
− 4
View file @ 2f2008b9
import logging
import pytest
from pydct.preprocess import zwise_median
from pydct.preprocess import zwise_reduce
from silx.io import get_data
from ._fixtures import darkend00_h5, darkend00_url, output_h5, output_url
zwise_median.logger.setLevel(logging.DEBUG)
zwise_reduce.logger.setLevel(logging.DEBUG)
# simple run to make sure there's no silly mistake
parser = zwise_median.parser
parser = zwise_reduce.parser
def test_parser_no_args():
......@@ -65,7 +65,7 @@ def main_kwargs(darkend00_url, output_url):
@pytest.mark.slow
def test_main(main_kwargs, output_h5):
zwise_median.main(**main_kwargs)
zwise_reduce.main(**main_kwargs)
# check outputs
path, _, __, data_urlstr = output_h5
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment