...
 
Commits (60)
......@@ -35,6 +35,8 @@ doc:
- python -m pip install silx --upgrade --pre
# for now we need to install nabu from sources
- install_nabu_master
- install_tomoscan
- install_nxtomomill
- python -m pip install jupyterlab
- python -m pip install Sphinx
- python -m pip install nbsphinx
......@@ -67,7 +69,8 @@ doc:
- source ./ci/install_scripts.sh
- install_orange3
- install_anyqt 'master'
- install_bliss
- install_tomoscan
- install_nxtomomill
- python -m pip install -r requirements-dev.txt
- python -m pip install -r requirements-test.txt
- python -m pip install fabio --upgrade --pre
......@@ -96,6 +99,7 @@ test:test-tomwer-tutorials:
- export ORANGE_WEB_LOG='False'
- python --version
- source ./ci/install_scripts.sh
- install_tomoscan
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install numpy --upgrade
......@@ -103,7 +107,6 @@ test:test-tomwer-tutorials:
- python -m pip install jupyter_client
- python -m pip install nbconvert
- python -m pip install ipykernel
- install_bliss
- python -m pip install -r requirements-dev.txt
- python -m pip install -r requirements-test.txt
- python -m pip install fabio --upgrade --pre
......
......@@ -22,12 +22,15 @@ Change Log
* add interface for data listener
* add interface for sinogram viewer
* axis: update interface
* viewer: add prototype of the next data viewer
* orangecontrib
* add sinogram widget: used to compute a sinogram from an edf or hdf5 (NXtomo) acquisition
* add data listener widget: used to connect with a tango device and retrieve information from it.
* add nabu widget: used to reconstruct using the nabu library
* add nxtomomill widget: used to convert from a bliss.hdf5 acquisition to a (NXTomo) .nx acquisition
* add prototype of the DataViewer
* add prototype of the new DataValidator (Move previous one to OldDataValidatorOW)
* app
* add nabu application: GUI for calling nabu or create nabu configuration file
......
......@@ -23,35 +23,26 @@ function silx_version(){
function install_nxtomomill(){
git clone --single-branch --branch master https://gitlab.esrf.fr/tomotools/nxtomomill.git
git clone --single-branch --branch 0.3 https://gitlab.esrf.fr/tomotools/nxtomomill.git
cd nxtomomill
python -m pip install .
cd ..
}
function install_orange3(){
git clone --single-branch --branch master https://github.com/payno/orange3.git
cd orange3
function install_tomoscan(){
git clone --single-branch --branch 0.3 https://gitlab.esrf.fr/tomotools/tomoscan.git
cd tomoscan
python -m pip install .
cd ..
}
function install_bliss(){
# install bliss with minimal dependacy
git clone --single-branch --branch master https://gitlab.esrf.fr/bliss/bliss.git
cd bliss
# move to the 1.3.0 tag
git checkout 1.3.0
git apply ../ci/patch_bliss.txt
python -m pip install louie
python -m pip install gevent
python -m pip install netifaces
python -m pip install treelib
python -m pip install tabulate
python -m pip install typeguard
python -m pip install cerberus
python -m pip install . --no-deps
function install_orange3(){
git clone --single-branch --branch master https://github.com/payno/orange3.git
cd orange3
python -m pip install .
cd ..
}
......
......@@ -78,9 +78,9 @@ author = 'ESRF'
# built documents.
#
# The short X.Y version.
version = '0.6'
version = '0.5'
# The full version, including alpha/beta/rc tags.
release = '0.6'
release = '0.5'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
......
......@@ -89,7 +89,9 @@ This is the list of actual widgets/boxes proposed by the workflow tool:
widgets/datalistener.rst
widgets/dataselectorwidget.rst
widgets/datatransfertwidget.rst
widgets/dataviewerwidget.rst
widgets/datavalidatorwidget.rst
widgets/olddatavalidatorwidget.rst
widgets/imagestackviewerwidget.rst
widgets/datawatcherwidget.rst
widgets/ftserieswidget.rst
......
data validator
``````````````
.. snapshotqt:: img/scanvalidatorwidget.png
import tempfile
from tomwer.test.utils import UtilsTest
from orangecontrib.tomwer.widgets.control.DataValidatorOW import DataValidatorOW
from tomwer.gui.viewerqwidget import ImageStackViewerValidator
from tomwer.core.scan.scanfactory import ScanFactory
from tomwer.core import utils
folder = UtilsTest.getInternalTestDir("001_0.28_19keV_Al63")
scan = ScanFactory.create_scan_object(folder)
widget = DataValidatorOW()
widget.addScan(scan)
widget.widget.stackImageViewerTab.setCurrentIndex(1)
widget.show()
.. image:: img/datavalidator.png
Widget displaying results of a reconstruction and asking to the user if he want to validate or not the reconstruction.
User can also ask for some modification on the reconstruction parameters.
......
data viewer
```````````
.. image:: img/dataviewer.png
Allow browsing through data (raw or normalized radios) and reconstructed slices.
(old) data validator
````````````````````
deprecated; use data validator instead.
.. snapshotqt:: img/oldscanvalidatorwidget.png
import tempfile
from tomwer.test.utils import UtilsTest
from orangecontrib.tomwer.widgets.control.DataValidatorOW import DataValidatorOW
from tomwer.gui.viewerqwidget import ImageStackViewerValidator
from tomwer.core.scan.scanfactory import ScanFactory
from tomwer.core import utils
folder = UtilsTest.getInternalTestDir("001_0.28_19keV_Al63")
scan = ScanFactory.create_scan_object(folder)
widget = DataValidatorOW()
widget.addScan(scan)
widget.widget.stackImageViewerTab.setCurrentIndex(1)
widget.show()
Widget displaying results of a reconstruction and asking to the user if he want to validate or not the reconstruction.
User can also ask for some modification on the reconstruction parameters.
Behavior
''''''''
Buttons
*******
* 'Change reconstruction parameters' will send a signal to the ftseries widget and ask for the user new reconstruction parameters (and then run again a reconstruction)
* 'Validate' will send a signal to the next connected boxes saying this data is ok for nect operations
This inherit from [ImageStackViewerWidget](imagestackviewer.html)
......@@ -159,16 +159,7 @@ class TestCopyNFolder(OrangeWorflowTest):
time.sleep(0.2)
self.assertTrue(self.dataHasBeenCopied())
signal_manager = self.canvas_window.current_document().scheme().signal_manager
timeout = self.TIMEOUT_TEST
while timeout > 0 and self.app.hasPendingEvents():
timeout = timeout - 0.2
self.app.processEvents()
print('blocking_nodes:', signal_manager.pending_nodes())
time.sleep(0.2)
if timeout <= 0.0:
raise TimeoutError('Loop infinitely on some qt event...')
self.app.processEvents()
def outpudirIsEmpty(self):
return len(os.listdir(self.outputdir)) == 0
......
This diff is collapsed.
......@@ -43,6 +43,7 @@ from silx.gui import qt
from tomwer.core import settings
from tomwer.core import utils
import functools
import copy
import logging
......@@ -242,12 +243,13 @@ class AxisOW(widget.OWWidget, WidgetLongProcessing):
def new_data_in(self, scan):
if scan is None:
return
scan_ = copy.copy(scan)
if not (settings.isOnLbsram() and utils.isLowOnMemory(
settings.get_lbsram_path())):
self._widget.setScan(scan=scan)
elif scan.axis_params is None:
scan.axis_params = QAxisRP()
self.process(scan=scan)
self._widget.setScan(scan=scan_)
elif scan_.axis_params is None:
scan_.axis_params = QAxisRP()
self.process(scan=scan_)
def process(self, scan):
if scan is None:
......
......@@ -41,6 +41,7 @@ from tomwer.core.scan.scanbase import TomwerScanBase
from tomwer.gui.reconstruction.darkref.darkrefcopywidget import DarkRefAndCopyWidget
from tomwer.synctools.ftseries import QReconsParams
from tomwer.web.client import OWClient
import copy
logger = logging.getLogger(__name__)
......@@ -128,7 +129,7 @@ class DarkRefAndCopyOW(widget.OWWidget, OWClient, WidgetLongProcessing):
if scanID is None:
return
assert isinstance(scanID, TomwerScanBase)
return self.widget.process(scanID)
return self.widget.process(copy.copy(scanID))
def signalReady(self, scanID):
assert isinstance(scanID, TomwerScanBase)
......@@ -143,6 +144,5 @@ class DarkRefAndCopyOW(widget.OWWidget, OWClient, WidgetLongProcessing):
return self.widget.recons_params
def close(self):
logger.info('close Dark refs')
self.widget.close()
super(DarkRefAndCopyOW, self).close()
......@@ -37,6 +37,7 @@ from tomwer.gui.reconstruction.ftserie import FtserieWidget
from tomwer.web.client import OWClient
from tomwer.core.scan.scanbase import TomwerScanBase
from silx.gui import qt
import copy
class FtseriesOW(widget.OWWidget, OWClient, WidgetLongProcessing):
......@@ -153,7 +154,10 @@ class FtseriesOW(widget.OWWidget, OWClient, WidgetLongProcessing):
if scan is None:
return
assert isinstance(scan, TomwerScanBase)
self._ftserie.pathReceived(scan)
scan_ = copy.copy(scan)
scan_.clear_latest_reconstructions()
self._ftserie.setScan(scan=scan_)
self._ftserie.pathReceived(scan_)
def _updateSettingsVals(self):
self._rpSetting = self._ftserie.recons_params.to_dict()
......
......@@ -45,6 +45,7 @@ from Orange.widgets import widget, gui
from Orange.widgets.widget import Input, Output
from orangecontrib.tomwer.orange.settings import CallbackSettingsHandler
from tomwer.web.client import OWClient
import copy
_logger = logging.getLogger(__name__)
......@@ -146,6 +147,8 @@ class NabuOW(widget.OWWidget, OWClient, WidgetLongProcessing):
# expose API
self.getConfiguration = self._nabuWidget.getConfiguration
self.setConfiguration = self._nabuWidget.setConfiguration
self.getMode = self._nabuWidget.getMode
self.setMode = self._nabuWidget.setMode
# connect signal / slot
self._processingStack.sigComputationStarted.connect(self._startProcessing)
......@@ -159,7 +162,11 @@ class NabuOW(widget.OWWidget, OWClient, WidgetLongProcessing):
assert isinstance(scan, (TomwerScanBase, type(None)))
if scan is None:
return
self._processingStack.add(scan, self.getConfiguration())
scan_ = copy.copy(scan)
scan_.clear_latest_reconstructions()
# update the reconstruction mode if possible
self._nabuWidget.setScan(scan_)
self._processingStack.add(scan_, self.getConfiguration())
@Inputs.reprocess
def reprocess(self, scan):
......
......@@ -31,6 +31,7 @@ __date__ = "10/01/2018"
import functools
import logging
import os
import copy
from silx.gui import qt
from ..utils import WidgetLongProcessing
from Orange.widgets import settings
......@@ -128,19 +129,20 @@ class TofuOW(widget.OWWidget, OWClient, WidgetLongProcessing):
@Inputs.data_in
def process(self, scan):
if scan is not None:
assert isinstance(scan, TomwerScanBase)
scan_ = copy.copy(scan)
self._executeButton.setEnabled(True)
self._lastScan = scan
self._mainWidget.loadFromScan(scan.path)
self._lastScan = scan_
self._mainWidget.loadFromScan(scan_.path)
recons_param = self._mainWidget.getParameters()
add_options = self._mainWidget.getAdditionalRecoOptions()
# TODO: should be recorded in self._viewer widget
remove_existing = self._mainWidget.removeOutputDir()
assert isinstance(scan, TomwerScanBase)
callback = functools.partial(self.Outputs.data_out.send, scan)
callback = functools.partial(self.Outputs.data_out.send, scan_)
self._reconsStack.add(recons_obj=LaminoReconstruction(),
scan_id=scan,
scan_id=scan_,
recons_params=recons_param,
additional_opts=add_options,
remove_existing=remove_existing,
......
......@@ -31,6 +31,7 @@ __date__ = "29/05/2017"
import unittest
from . import test_axis
from . import test_nabu_widget
from . import test_pyhst_widget
def suite():
......@@ -38,5 +39,6 @@ def suite():
test_suite.addTests([
test_axis.suite(),
test_nabu_widget.suite(),
test_pyhst_widget.suite(),
])
return test_suite
......@@ -219,9 +219,9 @@ class TestAxisStack(TestCaseQt):
self.qapp.processEvents()
self._mainWindow.new_data_in(scan)
self.assertEqual(self._scan1.axis_params.value, None)
self.assertEqual(self._scan2.axis_params.value, None)
self.assertEqual(self._scan3.axis_params.value, None)
self.assertEqual(self._scan1.axis_params, None)
self.assertEqual(self._scan2.axis_params, None)
self.assertEqual(self._scan3.axis_params, None)
def testUnlockStack(self):
"""Check that all axis position will be computed properly if we set a
......@@ -235,9 +235,9 @@ class TestAxisStack(TestCaseQt):
time.sleep(0.2)
self.qapp.processEvents()
self.assertNotEqual(self._scan1.axis_params.value, None)
self.assertNotEqual(self._scan2.axis_params.value, None)
self.assertNotEqual(self._scan3.axis_params.value, None)
self.assertNotEqual(self._scan1.axis_params, None)
self.assertNotEqual(self._scan2.axis_params, None)
self.assertNotEqual(self._scan3.axis_params, None)
def testLockStack(self):
"""Check that axis position will be simply copy if we are in a lock
......
......@@ -38,13 +38,16 @@ from silx.gui import qt
from orangecontrib.tomwer.widgets.reconstruction.NabuOW import NabuOW
from tomwer.core.scan.hdf5scan import HDF5TomoScan
from tomwer.synctools.ftseries import QReconsParams
from tomwer.core.process.reconstruction.nabu import _NabuMode
from tomwer.test.utils import UtilsTest
from silx.gui.utils.testutils import TestCaseQt
from silx.gui.utils.testutils import SignalListener
from tomwer.core.settings import mock_lsbram
from tomwer.core import utils
from tomoscan.scanbase import _FOV
from glob import glob
import time
import h5py
logging.disable(logging.INFO)
......@@ -86,20 +89,44 @@ class TestNabuWidget(TestCaseQt):
TestCaseQt.tearDown(self)
def testLowMemory(self):
"""Make sure no reconstruction is started if we are low in memory in
lbsram"""
self.assertEqual(len(glob(os.path.join(self.scan_dir, '*.cfg'))), 0)
self.widget.process(self.scan)
def wait_processing(self):
timeout = 10
while timeout >= 0 and self.signal_listener.callCount() == 0:
timeout -= 0.1
time.sleep(0.1)
if timeout <= 0.0:
raise TimeoutError('nabu widget never end processing')
def testLowMemory(self):
"""Make sure no reconstruction is started if we are low in memory in
lbsram"""
self.assertEqual(len(glob(os.path.join(self.scan_dir, '*.cfg'))), 0)
self.widget.process(self.scan)
self.wait_processing()
# make sure no processing was run
self.assertEqual(len(glob(os.path.join(self.scan_dir, '*.cfg'))), 0)
def patch_fov(self, value: str):
with h5py.File(self.scan.master_file, mode='a') as h5s:
for entry in ('entry0000', 'entry0001'):
entry_node = h5s[entry]
if 'instrument/detector/field_of_view' in entry_node:
del entry_node['instrument/detector/field_of_view']
entry_node['instrument/detector/field_of_view'] = value
def testSetConfiguration(self):
"""Make sure the configuration evolve from scan information"""
self.assertEqual(self.widget.getMode(), _NabuMode.FULL_FIELD)
self.patch_fov(value=_FOV.HALF.value)
self.widget.process(self.scan)
self.wait_processing()
self.assertEqual(self.widget.getMode(), _NabuMode.HALF_ACQ)
self.patch_fov(value=_FOV.FULL.value)
self.scan.clear_caches()
self.widget.process(self.scan)
self.wait_processing()
self.assertEqual(self.widget.getMode(), _NabuMode.FULL_FIELD)
def suite():
test_suite = unittest.TestSuite()
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "27/03/2020"
import logging
import os
import shutil
import tempfile
import unittest
from silx.gui import qt
from orangecontrib.tomwer.widgets.reconstruction.FtseriesOW import FtseriesOW
from tomwer.core.scan.hdf5scan import HDF5TomoScan
from tomwer.synctools.ftseries import QReconsParams
from tomwer.core.process.reconstruction.nabu import _NabuMode
from tomwer.test.utils import UtilsTest
from silx.gui.utils.testutils import TestCaseQt
from tomoscan.scanbase import _FOV
import time
import h5py
logging.disable(logging.INFO)
class TestPyHstWidget(TestCaseQt):
"""class testing the DarkRefWidget"""
def setUp(self):
TestCaseQt.setUp(self)
self._recons_params = QReconsParams()
self.widget = FtseriesOW(parent=None, _connect_handler=False)
self.scan_dir = tempfile.mkdtemp()
# create dataset
self.master_file = os.path.join(self.scan_dir,
'frm_edftomomill_twoentries.nx')
shutil.copyfile(UtilsTest.getH5Dataset(folderID='frm_edftomomill_twoentries.nx'),
self.master_file)
self.scan = HDF5TomoScan(scan=self.master_file, entry='entry0000')
def tearDown(self):
self._recons_params = None
self.widget.setAttribute(qt.Qt.WA_DeleteOnClose)
self.widget.close()
self.widget = None
TestCaseQt.tearDown(self)
def patch_fov(self, value: str):
with h5py.File(self.scan.master_file, mode='a') as h5s:
for entry in ('entry0000', 'entry0001'):
entry_node = h5s[entry]
if 'instrument/detector/field_of_view' in entry_node:
del entry_node['instrument/detector/field_of_view']
entry_node['instrument/detector/field_of_view'] = value
def testSetConfiguration(self):
"""Make sure the configuration evolve from scan information"""
axis_widget = self.widget._ftserie._recParamSetEditor._axisWidget
self.assertFalse(axis_widget._qcbHalfAcq.isChecked())
self.patch_fov(value=_FOV.HALF.value)
self.widget._ftserie.setScan(self.scan)
self.qapp.processEvents()
self.assertTrue(axis_widget._qcbHalfAcq.isChecked())
self.patch_fov(value=_FOV.FULL.value)
self.scan.clear_caches()
self.qapp.processEvents()
self.widget._ftserie.setScan(self.scan)
self.qapp.processEvents()
self.assertFalse(axis_widget._qcbHalfAcq.isChecked())
def suite():
test_suite = unittest.TestSuite()
for ui in (TestPyHstWidget, ):
test_suite.addTest(
unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest="suite")
......@@ -37,19 +37,13 @@ import os
from silx.gui import qt
from tomwer.core.utils import ignoreLowMemory
from tomwer.core import settings
try:
import bliss
except ImportError:
has_bliss = False
else:
from orangecontrib.tomwer.widgets.control.DataListenerOW import DataListenerOW
from orangecontrib.tomwer.test.utils import OrangeWorflowTest
from tomwer.test.utils import UtilsTest
from tomwer.synctools.datalistener import _mock_acquisition_info
from silx.gui.utils.testutils import TestCaseQt
from silx.gui.utils.testutils import SignalListener
from tomwer.test.utils import skip_gui_test, skip_orange_workflows_test
has_bliss = True
from orangecontrib.tomwer.widgets.control.DataListenerOW import DataListenerOW
from orangecontrib.tomwer.test.utils import OrangeWorflowTest
from tomwer.test.utils import UtilsTest
from tomwer.synctools.datalistener import _mock_acquisition_info
from silx.gui.utils.testutils import TestCaseQt
from silx.gui.utils.testutils import SignalListener
from tomwer.test.utils import skip_gui_test, skip_orange_workflows_test
logging.disable(logging.INFO)
......
......@@ -34,8 +34,9 @@ import unittest
from silx.gui import qt
from orangecontrib.tomwer.widgets.control.DataValidatorOW import \
DataValidatorOW
from orangecontrib.tomwer.widgets.control.OldDataValidatorOW import \
OldDataValidatorOW
from orangecontrib.tomwer.widgets.control.DataValidatorOW import DataValidatorOW
from tomwer.gui.qtapplicationmanager import QApplicationManager
from tomwer.core.scan.edfscan import EDFTomoScan
from tomwer.core.scan.scanbase import TomwerScanBase
......@@ -47,9 +48,9 @@ _qapp = QApplicationManager()
logging.disable(logging.INFO)
class TestAccumulation(unittest.TestCase):
class BaseTestAccumulation:
"""Test that all scans will be validated if many are send and no answer
is give until all received"""
is give until all received"""
NB_SCAN = 10
"""Number of scan to accumulate"""
......@@ -58,7 +59,7 @@ class TestAccumulation(unittest.TestCase):
N_RECONS = 2
DIM_MOCK_SCAN = 10
def setUp(self):
def _setUp(self):
self.scans = []
for iScan in range(TestAccumulation.NB_SCAN):
scanID = tempfile.mkdtemp()
......@@ -69,12 +70,9 @@ class TestAccumulation(unittest.TestCase):
dim=self.DIM_MOCK_SCAN)
self.scans.append(scan)
self.scanValidator = None
self.scanValidator = DataValidatorOW(None)
self.scanValidator._warnValManualShow = True
self.scanValidator.setAttribute(qt.Qt.WA_DeleteOnClose)
def tearDown(self):
def _tearDown(self):
_qapp.processEvents()
for f in self.scans:
if os.path.isdir(f.path):
......@@ -82,20 +80,46 @@ class TestAccumulation(unittest.TestCase):
self.scanValidator.close()
def test(self):
assert self.scanValidator is not None
for scan in self.scans:
self.scanValidator.addScan(scan)
_qapp.processEvents()
self.assertTrue(len(self.scanValidator._scansToValidate) == TestAccumulation.NB_SCAN)
self.assertEqual(self.scanValidator.getNScanToValidate(), TestAccumulation.NB_SCAN)
for scanID in self.scans:
self.scanValidator._validateScan(scanID)
_qapp.processEvents()
self.assertTrue(len(self.scanValidator._scansToValidate) == 0)
self.assertEqual(self.scanValidator.getNScanToValidate(), 0)
class TestAccumulation(unittest.TestCase, BaseTestAccumulation):
"""Test validation for the current data validator"""
def setUp(self):
BaseTestAccumulation._setUp(self)
self.scanValidator = DataValidatorOW(None)
self.scanValidator._warnValManualShow = True
self.scanValidator.setAttribute(qt.Qt.WA_DeleteOnClose)
def tearDown(self):
BaseTestAccumulation._tearDown(self)
class TestAccumulationOld(unittest.TestCase, BaseTestAccumulation):
def setUp(self):
BaseTestAccumulation._setUp(self)
self.scanValidator = OldDataValidatorOW(None)
self.scanValidator._warnValManualShow = True
self.scanValidator.setAttribute(qt.Qt.WA_DeleteOnClose)
def tearDown(self):
BaseTestAccumulation._tearDown(self)
class TestValidation(unittest.TestCase):
"""Test the validation command"""
class BaseTestValidation(unittest.TestCase):
"""Test validation for the old data validator"""
class _ValidationReceiver(qt.QObject):
def __init__(self):
qt.QObject.__init__(self)
......@@ -104,19 +128,6 @@ class TestValidation(unittest.TestCase):
def count(self):
self.counter = self.counter + 1
class _DataValidatorWidgetPatched(DataValidatorOW):
"""A DataValidatorOW without the Orange send signal process"""
scanReady = qt.Signal(TomwerScanBase)
_warnValManualShow = True
def _validated(self, scan):
"""Callback when the validate button is pushed"""
assert isinstance(scan, TomwerScanBase)
if scan is not None:
self.Outputs.data_out.send(scan)
self.scanReady.emit(scan)
del self._scansToValidate[scan.path]
@classmethod
def setUpClass(cls):
cls.rootdir = tempfile.mkdtemp()
......@@ -137,18 +148,55 @@ class TestValidation(unittest.TestCase):
shutil.rmtree(cls.rootdir)
def tearDown(self):
self.scanValidator.setAttribute(qt.Qt.WA_DeleteOnClose)
self.scanValidator.close()
del self.scanValidator
self.scanValidator = None
unittest.TestCase.tearDown(self)
def setUp(self):
unittest.TestCase.setUp(self)
self.scanValidator = TestValidation._DataValidatorWidgetPatched()
self.scanValidator = None
class TestValidation(BaseTestValidation):
"""Test validation for the latest data validation widget"""
def setUp(self):
BaseTestValidation.setUp(self)
self.scanValidator = DataValidatorOW()
self.scanValidator._warnValManualShow = True
self.scanValidator.scanReady.connect(self.validationReceiver.count)
self.scanValidator._widget.sigScanReady.connect(self.validationReceiver.count)
self.validationReceiver.counter = 0
self.scanValidator.setAttribute(qt.Qt.WA_DeleteOnClose)
def testAutomaticValidation(self):
for scan in self.scans:
self.scanValidator.addScan(EDFTomoScan(scan))
self.assertTrue(self.validationReceiver.counter == 0)
self.scanValidator.setAutomaticValidation(True)
while (_qapp.hasPendingEvents()):
_qapp.processEvents()
self.assertTrue(self.validationReceiver.counter == 2)
self.assertTrue(len(self.scanValidator._widget._scansToValidate) == 0)
class TestValidationOld(BaseTestValidation):
"""Test validation for the latest data validation widget"""
class _DataValidatorWidgetPatched(OldDataValidatorOW):
"""A DataValidatorOW without the Orange send signal process"""
scanReady = qt.Signal(TomwerScanBase)
_warnValManualShow = True
def _validated(self, scan):
"""Callback when the validate button is pushed"""
assert isinstance(scan, TomwerScanBase)
if scan is not None:
self.Outputs.data_out.send(scan)
self.scanReady.emit(scan)
del self._scansToValidate[scan.path]
def testAutomaticValidation(self):
for scan in self.scans:
self.scanValidator.addScan(EDFTomoScan(scan))
......@@ -160,10 +208,19 @@ class TestValidation(unittest.TestCase):
self.assertTrue(self.validationReceiver.counter == 2)
self.assertTrue(len(self.scanValidator._scansToValidate) == 0)
def setUp(self):
BaseTestValidation.setUp(self)
self.scanValidator = TestValidationOld._DataValidatorWidgetPatched()
self.scanValidator._warnValManualShow = True
self.scanValidator.scanReady.connect(self.validationReceiver.count)
self.validationReceiver.counter = 0
self.scanValidator.setAttribute(qt.Qt.WA_DeleteOnClose)
def suite():
test_suite = unittest.TestSuite()
for ui in (TestAccumulation, TestValidation):
for ui in (TestAccumulation, TestAccumulationOld, TestValidation,
TestValidationOld):
test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno", ]
__license__ = "MIT"
__date__ = "21/07/2020"
from silx.gui import qt
from Orange.widgets import widget, gui
from Orange.widgets.widget import Input
from tomwer.gui.visualization.dataviewer import DataViewer
from tomwer.core.scan.scanbase import TomwerScanBase
from Orange.widgets import settings
import logging
_logger = logging.getLogger(__name__)
class DataViewerOW(widget.OWWidget):
"""a data viewer able to:
- display slices (latest reconstructed if any)
- display radios with or without normalization
:param parent: the parent widget
"""
name = "data viewer"
id = "orange.widgets.tomwer.dataviewer"
description = "allow user too browse through data"
icon = "icons/eye.png"
priority = 70
category = "esrfWidgets"
keywords = ["tomography", "file", "tomwer", "acquisition", "validation"]
want_main_area = True
resizing_enabled = True
compress_signal = False
_viewer_config = settings.Setting(dict())
class Inputs:
data_in = Input(name='data', type=TomwerScanBase)
def __init__(self, parent=None):
widget.OWWidget.__init__(self, parent)
self._layout = gui.vBox(self.mainArea, self.name).layout()
self.viewer = DataViewer(parent=self)
self._layout.addWidget(self.viewer)
self._setSettings(settings=self._viewer_config)
# connect signal / slot
self.viewer.sigConfigChanged.connect(self._updateSettings)
@Inputs.data_in
def addScan(self, scan):
self.viewer.setScan(scan)
def sizeHint(self):
return qt.QSize(400, 500)
def _updateSettings(self):
self._viewer_config['mode'] = self.viewer.getDisplayMode()
self._viewer_config['slice_opt'] = self.viewer.getSliceOption()
self._viewer_config['radio_opt'] = self.viewer.getRadioOption()
def _setSettings(self, settings):
old_state = self.viewer.blockSignals(True)
if 'mode' in settings:
self.viewer.setDisplayMode(settings['mode'])
if 'slice_opt' in settings:
self.viewer.setSliceOption(settings['slice_opt'])
if 'radio_opt' in settings:
self.viewer.setRadioOption(settings['radio_opt'])
self.viewer.blockSignals(old_state)
......@@ -44,11 +44,11 @@ class ImageStackViewerOW(widget.OWWidget):
:param parent:the parent widget
:param FtserieReconstruction ftseries: the initial reconstruction to show
"""
name = "data viewer"
name = "(old) data viewer"
id = "orange.widgets.tomwer.imagestackviewer"
description = "Show a small recap of the reconstruction runned"
icon = "icons/eye.png"
priority = 70
icon = "icons/eyecrack.png"
priority = 700
category = "esrfWidgets"
keywords = ["tomography", "file", "tomwer", "acquisition", "validation"]
......
-r requirements.txt
requests-unixsocket
nabu
nxtomomill>=0.3.0
scipy
werkzeug
json-rpc
\ No newline at end of file
......@@ -477,7 +477,7 @@ def get_project_configuration(dry_run):
"Orange3",
"psutil",
"silx",
"tomoscan>=0.3",
"tomoscan>=0.3.1",
]
full_requires = [
......@@ -494,7 +494,7 @@ def get_project_configuration(dry_run):
'lxml',
'werkzeug',
'json-rpc',
'nxtomomill>=0.3',
'nxtomomill>=0.3.1',
'scipy',
]
......
......@@ -84,11 +84,11 @@ def main():
module_name="tomwer.app.sinogramviewer",
description="Allows to compute on the fly sinogram "
"and display them")
launcher.add_command("slicestack",
launcher.add_command("slice-stack",
module_name="tomwer.app.slicestack",
description="Display each reconstruction contained "
"in the ROOT_DIR or sub-folders")
launcher.add_command("radiostack",
launcher.add_command("radio-stack",
module_name="tomwer.app.radiostack",
description="Display each radio contained in the "
"ROOT_DIR or sub-folders")
......
......@@ -49,6 +49,7 @@ class NabuWindow(NabuDialog):
def setScan(self, scan):
self.__scan = scan
NabuDialog.setScan(self, scan)
def getScan(self):
return self.__scan
......
......@@ -9,6 +9,7 @@ import argparse
from tomwer.gui.utils.splashscreen import getMainSplashScreen
from tomwer.gui.stacks import RadioStack
from tomwer.gui import icons
from tomwer.core.scan.hdf5scan import HDF5TomoScan
import signal
logging.basicConfig()
......@@ -25,6 +26,8 @@ def addFolderAndSubFolder(stack, path):
_path = os.path.join(path, f)
if os.path.isdir(_path) is True:
addFolderAndSubFolder(stack, _path)
elif os.path.isfile(_path) and HDF5TomoScan.is_nexus_nxtomo_file(_path):
stack.addLeafScan(_path)
def sigintHandler(*args):
......@@ -56,7 +59,7 @@ def main(argv):
timer = qt.QTimer()
timer.start(500)
# Application have to wake up Python interpreter, else SIGINT is not
# catched
# cached
timer.timeout.connect(lambda: None)
splash = getMainSplashScreen()
......
......@@ -30,7 +30,7 @@ __date__ = "05/07/2017"
import os
import shutil
import fnmatch
from tomwer.core.process.baseprocess import SingleProcess, _input_desc, \
_output_desc
from tomwer.core.scan.scanfactory import ScanFactory
......@@ -140,6 +140,12 @@ class FolderTransfert(SingleProcess):
else:
_entry = None
output_scan = scan._deduce_transfert_scan(outputdir)
try:
output_scan._update_latest_recons_urls(old_path=scan.path,
new_path=output_scan.path)
except Exception as e:
logger.warning('Fail to convert url of latest reconstruction. Reason is:' + e)
output_scan.clear_latest_reconstructions()
self.__noticeTransfertSuccess(input_scan=scan,
output_scan=output_scan)
else:
......@@ -153,7 +159,13 @@ class FolderTransfert(SingleProcess):
self._signalCopying(scanID=source, outputdir=target)
output_scan = scan._deduce_transfert_scan(outputdir)
try:
output_scan._update_latest_recons_urls(old_path=scan.path,
new_path=output_scan.path)
except Exception as e:
logger.warning(
'Fail to convert url of latest reconstruction. Reason is:' + e)
output_scan.clear_latest_reconstructions()
RSyncManager().sync_file(source=source,
target=target,
wait=self._block,
......@@ -244,11 +256,9 @@ class FolderTransfert(SingleProcess):
mst_sample_file = DataListener.get_master_sample_file(
process_file=scan.process_file,
entry=scan.entry)
print('saving file found: {}'.format(mst_sample_file))
else:
mst_sample_file = None
if mst_sample_file is not None:
print('saving file destination will be {}'.format(self._get_master_sample_file_dst(mst_sample_file)))
files_sources.append(mst_sample_file)
files_dest.append(
self._get_master_sample_file_dst(mst_sample_file))
......@@ -263,6 +273,33 @@ class FolderTransfert(SingleProcess):
output_scan = HDF5TomoScan(scan=new_nx_file, entry=scan.entry)
else:
output_scan = None
# manage files generated (*slice*.h5/edf, *.cfg, *.par...)
# for reconstructed file, .h5, .edf if there is some conflict at one
# point I guess we might need to check file entry ? or rename the file
# according to the entry.
patterns = ['*.par', '*.cfg', '*.rec']
# manage .par, .cfg and .rec files if any
patterns += ['*slice*.hdf5', '*slice*.h5', '*slice*.edf']
# manage *slice*.hdf5 and *slice*.edf files (reconstructed slice)
patterns += ['tomwer_processes.h5', ]
def match_pattern(file_name):
file_name = file_name.lower()
for pattern in patterns:
if fnmatch.fnmatch(file_name, pattern):
return True
return False
dir_name = os.path.dirname(scan.master_file)
for file_ in os.listdir(dir_name):
if match_pattern(file_name=file_):
full_file_path = os.path.join(dir_name, file_)
files_sources.append(full_file_path)
files_dest.append(self._get_hdf5_sample_file_or_nx_dst(full_file_path))
delete_opt.append(True)
RSyncManager().sync_files(sources=files_sources,
targets=files_dest,
wait=self._block,
......
......@@ -63,7 +63,7 @@ pc_width = 50. / 100.
def compute_cor_nabu(radio_1: numpy.ndarray, radio_2: numpy.ndarray,
padding_mode, poly_deg=4, horz_fft_width=False,):
padding_mode, half_acq_cor_guess, horz_fft_width=False):
"""
Call nabu.preproc.alignement.CenterOfRotation.find_shift
......@@ -72,6 +72,11 @@ def compute_cor_nabu(radio_1: numpy.ndarray, radio_2: numpy.ndarray,
:param padding_mode:
:param poly_deg:
:param horz_fft_width:
:param half_acq_cor_guess: The approximate position of the rotation axis
from the image center. Optional. When given a
special algorithm is used which can work also
in half-tomo conditions.
:return:
"""
nabu_class = CenterOfRotation(horz_fft_width=horz_fft_width)
......@@ -79,7 +84,8 @@ def compute_cor_nabu(radio_1: numpy.ndarray, radio_2: numpy.ndarray,
img_2=numpy.fliplr(radio_2),
roi_yxhw=None,
padding_mode=padding_mode,
median_filt_shape=None)
median_filt_shape=None,
half_tomo_cor_guess=half_acq_cor_guess)
def compute_scan_cor_nabu(scan):
......@@ -98,109 +104,15 @@ def compute_scan_cor_nabu(scan):
_logger.info('compute scan axis from nabu CenterOfRotation with padding '
'mode %s' % scan.axis_params.padding_mode)
return compute_cor_nabu(radio_1=radio_1.copy(), radio_2=radio_2.copy(),
padding_mode=scan.axis_params.padding_mode)
def compute_scan_near(scan):
"""
will compute the near algorithm within the axis_params contains in the scan
and for the given scan
:param scan: scan for which we want to compute the center of rotation
:type: TomoBase
"""
assert scan.axis_params is not None
radio_1, radio_2 = AxisProcess.get_inputs(scan=scan)
if radio_1 is None or radio_2 is None:
raise NoAxisUrl('Unable to find projections for near axis calculation')
_logger.info('compute scan axis from near method for %s with near position'
': %f, window size: %s, fine step x: %s, look at stdmax: %s'
'' % (scan.path, scan.axis_params.near_position,
scan.axis_params.near_wx,
scan.axis_params.fine_step_x,
scan.axis_params.look_at_stdmax))
return compute_near(radio_1=radio_1.copy(), radio_2=radio_2.copy(),
near_pos=scan.axis_params.near_position,
window_size=scan.axis_params.near_wx,
fine_step_x=scan.axis_params.fine_step_x,
look_at_stdmax=scan.axis_params.look_at_stdmax)
def compute_near(radio_1: numpy.ndarray, radio_2: numpy.ndarray,
near_pos: float, window_size: float, fine_step_x: float,
look_at_stdmax: bool, flip_radio: bool = True) -> Union[None,float]:
"""
see :ref:`axis near algorithm` for a description of the algorithm
:param radio_1: normalized projection
:type: numpy.array
:param radio_2: normalized projection to be put in relation with radio 1
:type: numpy.array
:param near_pos: position to look around
:type: float
:param look_at_stdmax:
:param wxmin:
:type: float
:param wxmax:
:type: float
:return: center of rotation value in [-img_width/2, img_width/2]
:type: float
"""
mid_width = radio_1.shape[1] // 2
mid_height = radio_1.shape[0] // 2
beamheight = int(radio_1.shape[0] * pc_height)
wxmin = int(radio_1.shape[1] * (1. - pc_width)/2)
wxmax = radio_1.shape[1] - wxmin
wymin = int(mid_height - beamheight / 2)
wymax = int(mid_height + beamheight / 2)
# This is the distance to shift im2_flipped to make coincidence.
sh_im0 = int(round(2. * near_pos))
x_pos_inf = wxmin
x_pos_sup = wxmax
if look_at_stdmax is True:
xpos = get_stdmax_column(radio_1[mid_height,:])
if xpos <= mid_width:
x_pos_inf = window_size + 1
x_pos_sup = 2 * xpos - x_pos_inf
else:
x_pos_sup = radio_1.shape[1] - window_size - 1
x_pos_inf = 2 * xpos - x_pos_sup
assert x_pos_inf <= x_pos_sup
# define an integer window to shift im2_flipped around sh_im0
sh_int_range = sh_im0 + (numpy.arange(2*window_size+1)-window_size)
# define the number of fine steps in 1 pixel
n_fine_step_x = int(1./fine_step_x)
sh_fine_range = fine_step_x * numpy.arange(n_fine_step_x)
cor_f = sh_im0
h_minim = 1.e12
if flip_radio:
radio2_flipped = numpy.fliplr(radio_2.copy())
assert scan.axis_params.mode in (AxisMode.near, AxisMode.accurate)
if scan.axis_params.mode is AxisMode.accurate:
half_acq_cor_guess = None
else:
radio2_flipped = radio_2.copy()
progress = Progress(name='compute cor (using near)')
progress.setMaxAdvancement(len(sh_int_range))
for shint in sh_int_range:
imout0 = numpy.roll(radio2_flipped, shint, 1)
for shfine in sh_fine_range:
imout = img_shift(imout0, (0, shfine), order=1)
half_acq_cor_guess = scan.axis_params.near_position + scan.dim_1 // 2
diff = (radio_1 - imout)[wymin:wymax, x_pos_inf:x_pos_sup]
ampx = diff.max() - diff.min()
if ampx < h_minim:
h_minim = ampx
cor_f = shint + shfine
progress.increaseAdvancement(i=1)
return cor_f / 2
return compute_cor_nabu(radio_1=radio_1.copy(), radio_2=radio_2.copy(),
padding_mode=scan.axis_params.padding_mode,
half_acq_cor_guess=half_acq_cor_guess)
def get_stdmax_column(x: numpy.ndarray) -> float:
......@@ -268,7 +180,7 @@ class AxisProcess(SingleProcess):
RADIO_CALCULATIONS_METHODS = {
AxisMode.accurate: compute_scan_cor_nabu,
AxisMode.near: compute_scan_near,
AxisMode.near: compute_scan_cor_nabu,
}
def __init__(self, axis_params):
......
......@@ -46,11 +46,15 @@ from tomwer.core.process.baseprocess import (SingleProcess, _input_desc,
_output_desc)
from nabu.resources.dataset_analyzer import EDFDatasetAnalyzer, HDF5DatasetAnalyzer
from nabu.resources.dataset_analyzer import DatasetAnalyzer
from multiprocessing import Process
from multiprocessing import Process, Queue as _MQueue
from nabu.io.config import generate_nabu_configfile
from nabu import version as nabu_version
from silx.io.dictdump import h5todict
from silx.io.url import DataUrl
from contextlib import AbstractContextManager
import h5py
import logging
from logging.handlers import QueueHandler, QueueListener
_logger = TomwerLogger(__name__)
......@@ -79,10 +83,16 @@ def run_reconstruction(scan: TomwerScanBase, config: dict,
config['reconstruction']['rotation_axis_position'] = str(cor_nabu_ref)
nabu_configurations = _interpret_tomwer_configuration(config, scan=scan)
output_urls = []
for nabu_configuration in nabu_configurations:
l_config, slice_index = nabu_configuration
_run_single_reconstruction(config=l_config, scan=scan, local=local,
slice_index=slice_index, dry_run=dry_run)
output_urls.extend(_run_single_reconstruction(config=l_config,
scan=scan, local=local,
slice_index=slice_index,
dry_run=dry_run)
)
# tag latest reconstructions
scan.set_latest_reconstructions(output_urls)
class Nabu(SingleProcess):
......@@ -271,14 +281,37 @@ def _launch_reconstruction(config_file, slice_index):
proc = ProcessConfig(config_file)
from nabu.app.fullfield_cuda import CudaFullFieldPipeline
nabu_logger = logging.getLogger('nabu')
nabu_logger.setLevel(logging.INFO)
worker_process = CudaFullFieldPipeline(
proc,
sub_region=(None, None, slice_index, slice_index + 1),
logger=nabu_logger,
)