...
 
Commits (5)
The `process` application can process a workflow definde from a .ows file within a scan.
\ No newline at end of file
aliases = {
'orangecontrib.tomwer.widgets.reconstruction.DarkRefAndCopyOW.DarkRefAndCopyOW': 'tomwer.core.process.reconstruction.darkref.darkrefscopy.DarkRefsCopy',
'orangecontrib.tomwer.widgets.reconstruction.DarkRefAndCopyOW.DarkRefAndCopyOW': 'tomwer.core.process.reconstruction.darkref.darkrefs.DarkRefs',
'orangecontrib.tomwer.widgets.reconstruction.AxisOW.AxisOW': 'tomwer.core.process.reconstruction.axis.AxisProcess',
'orangecontrib.tomwer.widgets.reconstruction.NabuOW.NabuOW': 'tomwer.core.process.reconstruction.nabu.Nabu',
'orangecontrib.tomwer.widgets.control.NXTomomillOW.NXTomomillOW': 'tomwer.core.process.nxtomomill.NxTomomillProcess',
}
# aliases used to avoid executing orange widgets directly (and avoid Qt stuff...)
\ No newline at end of file
# aliases used to avoid executing orange widgets directly (and avoid Qt stuff...)
# NOTES:
#
# - dark ref copy is not managed by the process application. For now we don't have any process consistancy and we have to create each time a new process.
#
\ No newline at end of file
......@@ -98,6 +98,10 @@ def main():
launcher.add_command("nabu",
module_name="tomwer.app.nabuapp",
description="Use nabu to reconstruct a given scan")
launcher.add_command("process",
module_name="tomwer.app.process",
description="Process a given scan with a defined"
"workflow")
launcher.add_command("test",
module_name="tomwer.app.test_",
description="Launch tomwer unittest")
......
......@@ -109,8 +109,8 @@ class ToFuDialog(qt.QDialog):
self.tofuWidget.setEnabled(False)
qt.QApplication.processEvents()
qt.QApplication.setOverrideCursor(qt.Qt.WaitCursor)
self._reconsProcess.setProperties((preprocess_add_options,
reco_add_options))
self._reconsProcess.set_properties((preprocess_add_options,
reco_add_options))
self._reconsProcess.delete_existing = delete_existing
self._reconsProcess.reconstruction_parameters = recons_param
if self._dry_run:
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import sys
import argparse
from pypushflow.representation.scheme.ows_parser import OwsParser
from tomwer.core.scan.scanfactory import ScanFactory
from pypushflow.Workflow import ProcessableWorkflow
from tomwer.core.scan.blissscan import BlissScan
import signal
logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(__name__)
def _exec(scheme, scan=None, timeout=None):
"""
:param scheme:
:param scan:
:param timeout:
:return:
"""
# set up workflow
if len(scheme.start_nodes()) is 0:
_logger.warning('no start nodes found. Enable to process')
return None
mess = ' '.join(('start processing of scheme \'{}\''.format(scheme.title or 'unknow'),
'with', str(scan)))
_logger.info(mess)
workflow = ProcessableWorkflow(scheme=scheme)
# add SIGINT capture
def signal_handler(sig, frame):
_logger.warning('stop workflow execution on user request')
workflow._end_actor.join(0)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
workflow._start_actor.trigger(('data', scan.to_dict()))
workflow._end_actor.join(timeout)
res = workflow._end_actor.out_data
title = scheme.title or 'unknow'
_logger.info('workflow \'{}\' completed with {}'.format(title, str(scan)))
return res
def main(argv):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'workflow_file',
help='Path to the .ows file defining the workflow to process with the'
'provided scan')
parser.add_argument(
'scan_path',
help='Path to data to be processes (master file if come from an hdf5 '
'acquisition or EDF files folder if come from an EDF acquisition)')
parser.add_argument(
'--entry',
default=None,
help='An entry should be specify for hdf5 files')
parser.add_argument(
'--timeout',
default=None,
help='Timeout for the workflow execution')
# parser.add_argument(
# '--output-dir',
# help='output directory of the reconstruction(s)')
# parser.add_argument(
# '--debug',
# dest="debug",
# action="store_true",
# default=False,
# help='Set logging system in debug mode')
# parser.add_argument(
# '--dry-run',
# dest="dry_run",
# action="store_true",
# default=False,
# help='Only create the .nabu file and print the configuration used')
options = parser.parse_args(argv[1:])
if options.entry is not None:
scan = ScanFactory.create_scan_object(options.scan_path,
entry=options.entry,
accept_bliss_scan=True)
scans = (scan, )
else:
scans = ScanFactory.create_scan_objects(options.scan_path,
accept_bliss_scan=True)
if len(scans) > 1:
_logger.info('More than one scan found. Will process every scans')
# tune the log level
logging.getLogger('pypushflow').setLevel(logging.INFO)
scheme = OwsParser.scheme_load(options.workflow_file, load_handlers=True)
for scan in scans:
_exec(scheme=scheme, scan=scan, timeout=options.timeout)
if __name__ == '__main__':
main(sys.argv)
......@@ -86,10 +86,11 @@ class BaseProcess(object):
self._return_dict = False
"""should the return type of the handler should be TomoBase instance
objects or dict"""
self._settings = {}
# TODO: rename set_properties
def setProperties(self, properties):
pass
def set_properties(self, properties):
# call by pypushflow to define the process properties
raise NotImplementedError('Set properties not defined')
@staticmethod
def properties_help():
......
......@@ -135,7 +135,7 @@ class DataListener(SingleProcess):
del self._listening_thread
self._listening_thread = None
def setProperties(self, properties):
def set_properties(self, properties):
pass
def process(self, scan=None):
......
......@@ -80,7 +80,7 @@ class FolderTransfert(SingleProcess):
self._copying = False
self._block = False
def setProperties(self, properties):
def set_properties(self, properties):
# No properties stored for now
if 'dest_dir' in properties:
self.setDestDir(properties['dest_dir'])
......
......@@ -368,7 +368,7 @@ class _DataWatcher(EndlessProcess):
"""Update the progress bar"""
pass
def setProperties(self, properties):
def set_properties(self, properties):
for observe_folder_alias in ('observed_folder', 'folderObserved'):
if observe_folder_alias in properties:
self.setFolderObserved(properties[observe_folder_alias])
......
......@@ -50,7 +50,6 @@ class NxTomomillProcess(SingleProcess):
outputs = [_output_desc(name='data', type=TomwerScanBase, doc='scan path'), ]
@staticmethod
def deduce_output_file_path(master_file_name, entry):
file_dir = os.path.dirname(master_file_name)
......@@ -69,22 +68,25 @@ class NxTomomillProcess(SingleProcess):
def process(self, scan=None):
if scan is None:
return
if isinstance(scan, dict):
scan = BlissScan.from_dict(scan)
assert isinstance(scan, BlissScan)
output_file_path = self.deduce_output_file_path(master_file_name=scan.master_file,
entry=scan.entry)
_logger.info(' '.join(('write', str(scan), 'to', output_file_path)))
try:
conv = nxtomomill_converter.h5_to_nx(input_file_path=scan.master_file,
output_file=output_file_path,
entries=(scan.entry,),
single_file=True,
ask_before_overwrite=True,
ask_before_overwrite=False,
file_extension='.nx')
except Exception as e:
_logger.error(
'Fail to convert from bliss file: %s to NXTomo.'
'Conversion error is: %s' % (scan.master_file, e))
return None
else:
if len(conv) != 1:
_logger.error(
......@@ -94,6 +96,13 @@ class NxTomomillProcess(SingleProcess):
else:
conv_file, conv_entry = conv[0]
scan_converted = HDF5TomoScan(scan=conv_file, entry=conv_entry)
mess = "sending one scan %s" % str(scan_converted)
_logger.debug(mess),
_logger.info(
' '.join((str(scan), 'has been write at', str(scan_converted)))
)
_logger.info('---------------------------')
_logger.info(scan_converted.to_dict())
return scan_converted
def set_properties(self, properties):
# for now the NXProcess cannot be tune
pass
......@@ -315,6 +315,14 @@ class AxisProcess(SingleProcess):
"""
if scan is None:
return
if isinstance(scan, TomwerScanBase):
scan = scan
elif isinstance(scan, dict):
scan = ScanFactory.create_scan_object_frm_dict(scan)
else:
raise ValueError('input type {} is not managed'.format(scan))
_logger.info('start axis calculation for %s' % scan.path)
cor = error = None
try:
......
......@@ -166,7 +166,7 @@ class DarkRefs(SingleProcess, Queue):
self.sigScanReady.emit(scan)
self.execNext()
def process(self, scan):
def process(self, scan=None):
if scan is None:
return
......@@ -207,8 +207,8 @@ class DarkRefs(SingleProcess, Queue):
def execNext(self):
"""Launch the next reconstruction if any
"""
# try catch is needed because a signal can still be emitted event if the
# QObject has been destroyed.
# try catch is needed because a signal can still be emitted event if
# the QObject has been destroyed.
try:
if super(DarkRefs, self).empty():
return
......@@ -233,6 +233,7 @@ class DarkRefs(SingleProcess, Queue):
if self._forceSync is True:
self.worker.wait()
else:
logger.warning('#############################')
self.worker.process()
def _initWorker(self, scan, params, file_ext):
......@@ -321,12 +322,11 @@ class DarkRefs(SingleProcess, Queue):
- dark: 'None', 'Median', 'Average' \n
"""
def setProperties(self, properties):
# No properties stored for now
if 'dark' in properties:
self._recons_params.dark_calc_method = properties['dark']
if 'refs' in properties:
self._recons_params.ref_calc_method = properties['refs']
def set_properties(self, properties):
if '_rpSetting' in properties:
self._recons_params.load_from_dict(properties['_rpSetting'])
else:
self._recons_params.load_from_dict(properties)
@staticmethod
def get_darks_frm_process_file(process_file, entry=None,
......@@ -482,9 +482,7 @@ class DarkRefsWorker(OWClient):
logger.processSkipped(mess)
return
if not(self.scan and
os.path.exists(self.scan.path) and
os.path.isdir(self.scan.path)):
if not(self.scan and os.path.exists(self.scan.path)):
logger.warning("folder %s is not existing" % self.scan.path)
return
whats = (DarkRefs.WHAT_REF, DarkRefs.WHAT_DARK)
......@@ -493,6 +491,7 @@ class DarkRefsWorker(OWClient):
self._originalsDark = []
self._originalsRef = []
self.compute(scan=self.scan, what=what, mode=mode)
if self.scan.process_file is not None:
results = {}
interpretations = {}
......@@ -568,6 +567,7 @@ class DarkRefsWorker(OWClient):
if len(current_serie) > 0:
series.append(current_serie)
return series
logger.warning('AAAAAAAAAAAAAAAAAAAAAAAAA')
if mode is dkrf_reconsparams.Method.median:
method_ = numpy.median
......@@ -579,6 +579,7 @@ class DarkRefsWorker(OWClient):
if len(raw_series) == 0:
logger.info('No %s found for %s' % (what, scan))
return
logger.warning('BBBBBBBBBBBBBBBBBBBBBBBBBBBBB')
def load_data_serie(urls):
......@@ -587,12 +588,12 @@ class DarkRefsWorker(OWClient):
for url in cpt_slices.values():
path = url.file_path(), url.data_path(), str(url.data_slice())
url_set[path] = url
n_elmts = 0
for url in url_set.values():
my_slice = url.data_slice()
n_elmts += my_slice.stop - my_slice.start
data = None
start_z = 0
for url in url_set.values():
......@@ -625,7 +626,7 @@ class DarkRefsWorker(OWClient):
# res: index: sequence when the serie was taken
for serie_ in raw_series:
if what == 'dark' and len(res) >0:
if what == 'dark' and len(res) > 0:
continue
serie_index = min(serie_)
serie_data = load_data_serie(serie_)
......
......@@ -48,6 +48,7 @@ from tomwer.core.scan.scanbase import TomwerScanBase
from tomwer.core.scan.hdf5scan import HDF5TomoScan
from tomwer.core.scan.edfscan import EDFTomoScan
from tomwer.core.signal import Signal
from tomwer.core.process.baseprocess import _input_desc, _output_desc
from .settings import DARKHST_PREFIX, REFHST_PREFIX
logger = TomwerLogger(__name__)
......@@ -57,6 +58,12 @@ class DarkRefsCopy(DarkRefs):
"""
Reimplement Dark ref to deal with copy when there is no median/mean files
"""
inputs = [_input_desc(name='data', type=TomwerScanBase, handler='process',
doc='scan path'), ]
outputs = [_output_desc(name='data', type=TomwerScanBase, doc='scan path'), ]
def __init__(self, reconsparams=None):
super(DarkRefsCopy, self).__init__(reconsparams=reconsparams)
self.refPrefix = REFHST_PREFIX
......@@ -80,9 +87,8 @@ class DarkRefsCopy(DarkRefs):
def setModeAuto(self, auto):
self.worker.set_mode_auto(auto)
def setProperties(self, properties):
# No properties stored for now
pass
def set_properties(self, properties):
super().set_properties(properties=properties)
def get_dark_save_file(self):
return self.worker._dark_save_file
......@@ -239,22 +245,11 @@ class DarkRefsCopyWorker(DarkRefsWorker):
if copy_dark:
dark_info = self._get_dark_info()
if dark_info is not None:
configuration = {
'tomwer_info': dark_info['tomwer_info'],
'original scan': dark_info['original_scan'],
'srcurrent': dark_info['srcurrent'],
}
results['darks'] = {'0': dark_info['data']}
shape_dark = dark_info['data'].shape
if copy_flats:
flat_info = self._get_flat_info()
if flat_info is not None:
configuration = {
'tomwer_info': flat_info['tomwer_info'],
'original scan': flat_info['original_scan'],
'srcurrent': flat_info['srcurrent'],
}
flats = {'0': flat_info['data_start']}
shape_flat = flat_info['data_start'].shape
index_last_projection = max(scan.projections) or None
......@@ -283,14 +278,17 @@ class DarkRefsCopyWorker(DarkRefsWorker):
if incoherent_with_flat or incoherent_with_dark:
return
with scan.acquire_process_file_lock():
DarkRefs._register_process(process_file=scan.process_file,
entry=scan.entry,
process=DarkRefs,
configuration=configuration,
results=results,
process_index=self.scan.pop_process_index(),
overwrite=True)
# logger.warning('========================================')
# logger.warning('results are', results)
# logger.warning('configuration is', self.recons_params.to_dict())
# with scan.acquire_process_file_lock():
# DarkRefs._register_process(process_file=scan.process_file,
# entry=scan.entry,
# process=DarkRefs,
# configuration=self.recons_params.to_dict(),
# results=results,
# process_index=self.scan.pop_process_index(),
# overwrite=True)
def _copy_dark_to(self, scan):
"""The dark is copied without normalization"""
......@@ -535,7 +533,8 @@ class DarkRefsCopyWorker(DarkRefsWorker):
return None
def __del__(self):
shutil.rmtree(self._savedir)
if os.path.exists(self._savedir):
shutil.rmtree(self._savedir)
def clean_save_files(self):
for f in os.listdir(self._savedir):
......
......@@ -100,7 +100,7 @@ class _Ftseries(SingleProcess):
self.reconsStack.sigReconsMissParams.connect(self.updateReconsParam)
self.reconsStack.sigReconsFinished.connect(self._signalReconsReady)
def setProperties(self, properties):
def set_properties(self, properties):
if '_rpSetting' in properties:
self.recons_params.setStructs(properties['_rpSetting'])
......@@ -111,7 +111,7 @@ class _Ftseries(SingleProcess):
assert isinstance(recons_param, ReconsParams)
self.recons_params = recons_param
# TODO: this should be update to use am update instead
self.setProperties(self.recons_params)
self.set_properties(self.recons_params)
def setH5Exploration(self, b):
self._exploreForH5File = b
......
......@@ -734,7 +734,7 @@ class LaminoReconstruction(SingleProcess):
else:
return _scan
def setProperties(self, properties):
def set_properties(self, properties):
assert isinstance(properties, (tuple, list))
self.additional_preprocess_options, self.additional_reco_options = properties
......
......@@ -46,6 +46,7 @@ from tomwer.core.process.baseprocess import (SingleProcess, _input_desc,
_output_desc)
from nabu.resources.dataset_analyzer import EDFDatasetAnalyzer, HDF5DatasetAnalyzer
from nabu.resources.dataset_analyzer import DatasetAnalyzer
from tomwer.core.scan.scanfactory import ScanFactory
from multiprocessing import Process
from nabu.io.config import generate_nabu_configfile
from nabu import version as nabu_version
......@@ -106,20 +107,29 @@ class Nabu(SingleProcess):
def process(self, scan=None):
if scan is None:
return None
if isinstance(scan, TomwerScanBase):
scan = scan
elif isinstance(scan, dict):
scan = ScanFactory.create_scan_object_frm_dict(scan)
else:
run_reconstruction(scan=scan, config=self.get_configuration(),
dry_run=self.dry_run)
# register result
entry = 'entry'
if isinstance(scan, HDF5TomoScan):
entry = scan.entry
with scan.acquire_process_file_lock():
self.register_process(process_file=scan.process_file,
entry=entry,
configuration=self.get_configuration(),
results={},
process_index=scan.pop_process_index(),
overwrite=True)
raise ValueError('input type {} is not managed'.format(scan))
run_reconstruction(scan=scan, config=self.get_configuration(),
dry_run=self.dry_run)
# register result
entry = 'entry'
if isinstance(scan, HDF5TomoScan):
entry = scan.entry
with scan.acquire_process_file_lock():
self.register_process(process_file=scan.process_file,
entry=entry,
configuration=self.get_configuration(),
results={},
process_index=scan.pop_process_index(),
overwrite=True)
def pathReceived(self, scan):
return self.process(scan=scan)
@staticmethod
def program_name():
......
......@@ -70,7 +70,7 @@ class _ScanList(BaseProcess):
res.append(scan)
return res
def setProperties(self, properties):
def set_properties(self, properties):
if '_scanIDs' in properties:
self.setScanIDs(properties['_scanIDs'])
else:
......
......@@ -220,7 +220,7 @@ class ScanValidator(BaseProcess):
del self._scansToValidate[str(_ftserie)]
self._sendUpdateReconsParam(_TomwerBaseDock(tomo_instance=_ftserie))
def setProperties(self, properties):
def set_properties(self, properties):
# no properties/settings to be loaded
pass
......
......@@ -42,10 +42,16 @@ class BlissScan:
:warning: BlissScan is not compatible with tomwer treatment. This is
why it does not inherit from TomwerScanBase. This is a utility class.
"""
_TYPE = 'bliss_hdf5'
def __init__(self, master_file, entry, proposal_file,
scan_numbers=None):
self._master_file = master_file
self._entry = entry
if isinstance(entry, str) and not entry.startswith('/'):
self._entry = '/' + entry
else:
self._entry = entry
self._proposal_file = proposal_file
self._scan_numbers = scan_numbers or []
self._tomo_n = None
......@@ -129,3 +135,29 @@ class BlissScan:
entry=entry):
res.append(entry)
return tuple(res)
def to_dict(self):
return {
'DICT_TYPE_KEY': self._TYPE,
'master_file': self.master_file,
'entry': self.entry,
'proposal_file': self.proposal_file,
'scan_numbers': self.scan_numbers,
}
@staticmethod
def from_dict(ddict):
master_file = ddict['master_file']
entry = ddict['entry']
return BlissScan(master_file=master_file, entry=entry, proposal_file=None).load_frm_dict(ddict=ddict)
def load_frm_dict(self, ddict):
if 'master_file' in ddict:
self._master_file = ddict['master_file']
if 'entry' in ddict:
self._entry = ddict['entry']
if 'proposal_file' in ddict:
self._proposal_file = ddict['proposal_file']
if 'scan_numbers' in ddict:
self._scan_numbers = ddict['scan_numbers']
return self
\ No newline at end of file
......@@ -117,7 +117,10 @@ class HDF5TomoScan(_tsHDF5TomoScan, TomwerScanBase):
@staticmethod
def from_dict(_dict):
scan = HDF5TomoScan(scan=None)
path = _dict[HDF5TomoScan.DICT_PATH_KEY]
entry = _dict[HDF5TomoScan._DICT_ENTRY_KEY]
scan = HDF5TomoScan(scan=path, entry=entry)
scan.load_from_dict(_dict=_dict)
return scan
......@@ -132,18 +135,20 @@ class HDF5TomoScan(_tsHDF5TomoScan, TomwerScanBase):
data = json.load(_dict)
else:
data = _dict
if not (self._DICT_TYPE_KEY in data and data[self._DICT_TYPE_KEY] == self._TYPE):
if not (self.DICT_TYPE_KEY in data and data[self.DICT_TYPE_KEY] == self._TYPE):
raise ValueError('Description is not an EDFScan json description')
_tsHDF5TomoScan.load_from_dict(self)
_tsHDF5TomoScan.load_from_dict(self, _dict)
assert self._DICT_PATH_KEY in data
assert self._DICT_LAMINO_RP_KEY in data
self.path = data[self._DICT_PATH_KEY]
recons_param_data = data[self._DICT_TOMO_RP_KEY]
self.path = data[self.DICT_PATH_KEY]
if self._DICT_TOMO_RP_KEY in data:
recons_param_data = data[self._DICT_TOMO_RP_KEY]
else:
recons_param_data = None
if recons_param_data is not None:
self.ftseries_recons_params = ReconsParams.from_dict(recons_param_data)
self.lamino_recons_params = data[self._DICT_LAMINO_RP_KEY]
if self._DICT_LAMINO_RP_KEY in data:
self.lamino_recons_params = data[self._DICT_LAMINO_RP_KEY]
return self
def update(self):
......
......@@ -119,13 +119,13 @@ class ScanFactory(object):
@staticmethod
def create_scan_object_frm_dict(_dict):
if TomoScanBase.DICT_TYPE_KEY not in _dict:
raise ValueError('given dict is not recognized. Cannot find'
'', TomoScanBase.DICT_TYPE_KEY)
raise ValueError('given dict is not recognized. Cannot find {}'.format(TomoScanBase.DICT_TYPE_KEY))
elif _dict[TomoScanBase.DICT_TYPE_KEY] == EDFTomoScan._TYPE:
return EDFTomoScan(scan=None).load_from_dict(_dict)
elif _dict[TomoScanBase.DICT_TYPE_KEY] == HDF5TomoScan._TYPE:
return HDF5TomoScan.from_dict(_dict)
else:
raise ValueError('Scan type', _dict[TomoScanBase.DICT_TYPE_KEY],
'is not managed')
raise ValueError('Scan type {} is not managed'.format(_dict[TomoScanBase.DICT_TYPE_KEY]))
@staticmethod
def is_tomo_scandir(scan_path):
......
......@@ -74,9 +74,6 @@ class BlissSequenceRSyncWorker(RSyncWorker):
self._dst_bliss_master_file = dst_proposal_file
self._bliss_sample_file = src_sample_file
self._dst_bliss_sample_file = dst_sample_file
print('***********')
print('master sample file is', src_sample_file)
print('destination saving file is', dst_sample_file)
def _process_sync(self):
super(BlissSequenceRSyncWorker, self)._process_sync()
......