Tango timeout for creating a directory is 3 sec and needs to be increased, configurable and optionally disabled
File system issues cause any operation make makedirs to take many seconds. The internal writer monitoring has the timeout set to 10 sec:
However when calling SCAN_SAVING.create_path
the default tango timeout of 3 sec since it does not use the _retry
logic.
So the solution is to not use self._proxy
directly but use the retry wrapper. In addition we should probably refactor this code since too complicated. We also might want to option to configure the timeout and optionally disable. We currently do this at ID22 and all runs fine, even in the presence of file system latency.
2024-04-16 16:24:03,196 tdxrd exceptions ERROR : Traceback (most recent call last):
File "/users/opid11/bliss_scripts/user_exp/ma6050.py", line 17, in run_exp
diff_small_rotation()
@bs_hg: 1.3
@bs_vg: 0.5
@prefix: 'f160N'
File "/users/opid11/bliss_scripts/user_exp/ma6050.py", line 46, in diff_small_rotation
finterlaced(diffrz, -10, 0.05, 400, 0.09)
File "/home/blissadm/local/fscan.git/fscan/finterlaced.py", line 414, in __call__
self.run_with_pars(pars)
@acq_size: 0.05
@acq_time: 0.09
@kwargs: {}
@mode: 'ZIGZAG'
@motor: <bliss.common.axis.Axis object at 0x7f1abde13940>
@npoints: 400
@pars: {'motor': <bliss.common.axis.Axis object at 0x7f1abde13940>, 'start_pos': -10, 'acq_size': 0.05, 'npoints': 400, 'acq_time': 0.09, 'mode': 'ZIGZAG'}
@self: <id11.scans.fscans.ID11FInterlaced object at 0x7f1a583d89a0>
@start_pos: -10
File "/home/blissadm/local/fscan.git/fscan/fscanrunner.py", line 77, in run_with_pars
self.run(scan_info)
@pars: {'motor': <bliss.common.axis.Axis object at 0x7f1abde13940>, 'start_pos': -10, 'acq_size': 0.05, 'npoints': 400, 'acq_time': 0.09, 'mode': 'ZIGZAG'}
@scan_info: None
@self: <id11.scans.fscans.ID11FInterlaced object at 0x7f1a583d89a0>
@user_pars: {'motor': <bliss.common.axis.Axis object at 0x7f1abde13940>, 'start_pos': -10, 'acq_size': 0.05, 'npoints': 400, 'acq_time': 0.09, 'mode': 'ZIGZAG'}
File "/home/blissadm/local/fscan.git/fscan/fscanrunner.py", line 196, in run
self.start()
@scan_info: None
@self: <id11.scans.fscans.ID11FInterlaced object at 0x7f1a583d89a0>
File "/home/blissadm/local/fscan.git/fscan/fscanrunner.py", line 66, in start
self._scan.run()
@self: <id11.scans.fscans.ID11FInterlaced object at 0x7f1a583d89a0>
File "/home/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1289, in run
self._execute_scan_runner(runner)
@add_context: <function Scan.run.<locals>.add_context at 0x7f19883c3a60>
@capture: <function capture_exceptions.<locals>.capture at 0x7f19883c3f70>
@cstack: <contextlib.ExitStack object at 0x7f198a726730>
@ctx: <contextlib._GeneratorContextManager object at 0x7f197ebba1c0>
@runner: <bliss.scanning.scan._ScanIterationsRunner object at 0x7f198af26850>
@self: Scan(number=1, name=finterlaced, path=/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/L1_loading_f160N_bigbeam.h5)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/contextlib.py", line 532, in __exit__
raise exc_details[1]
@_fix_exception_context: <function ExitStack.__exit__.<locals>._fix_exception_context at 0x7f19903c84c0>
@cb: <bound method _GeneratorContextManager.__exit__ of <contextlib._GeneratorContextManager object at 0x7f198a7269d0>>
@exc_details: ... (truncated)
@fixed_ctx: None
@frame_exc: ... (truncated)
@is_sync: True
@new_exc_details: ... (truncated)
@pending_raise: True
@received_exc: True
@self: <contextlib.ExitStack object at 0x7f198a726730>
@suppressed_exc: True
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/contextlib.py", line 517, in __exit__
if cb(*exc_details):
@_fix_exception_context: <function ExitStack.__exit__.<locals>._fix_exception_context at 0x7f19903c84c0>
@cb: <bound method _GeneratorContextManager.__exit__ of <contextlib._GeneratorContextManager object at 0x7f198a7269d0>>
@exc_details: ... (truncated)
@fixed_ctx: None
@frame_exc: ... (truncated)
@is_sync: True
@new_exc_details: ... (truncated)
@pending_raise: True
@received_exc: True
@self: <contextlib.ExitStack object at 0x7f198a726730>
@suppressed_exc: True
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/contextlib.py", line 126, in __exit__
next(self.gen)
@self: <contextlib._GeneratorContextManager object at 0x7f198a7269d0>
@traceback: None
@typ: None
@value: None
File "/home/blissadm/local/bliss.git/bliss/common/cleanup.py", line 290, in capture_exceptions
raise value
@capture: <function capture_exceptions.<locals>.capture at 0x7f19883c3160>
@etype: <class 'PyTango.CommunicationFailed'>
@excepthook: <built-in function excepthook>
@infos: ... (truncated)
@raise_index: 0
@tb: None
@value: None
File "/home/blissadm/local/bliss.git/bliss/common/cleanup.py", line 266, in capture
yield
@excepthook: <built-in function excepthook>
@infos: ... (truncated)
@raise_index: 0
File "/home/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1374, in wrapper
yield
@capture: <function capture_exceptions.<locals>.capture at 0x7f19883c3160>
File "/home/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1289, in run
self._execute_scan_runner(runner)
@add_context: <function Scan.run.<locals>.add_context at 0x7f19883c3a60>
@capture: <function capture_exceptions.<locals>.capture at 0x7f19883c3f70>
@cstack: <contextlib.ExitStack object at 0x7f198a726730>
@ctx: <contextlib._GeneratorContextManager object at 0x7f197ebba1c0>
@runner: <bliss.scanning.scan._ScanIterationsRunner object at 0x7f198af26850>
@self: Scan(number=1, name=finterlaced, path=/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/L1_loading_f160N_bigbeam.h5)
File "/home/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1304, in _execute_scan_runner
self.prepare(self.scan_info, self.acq_chain._tree)
@runner: <bliss.scanning.scan._ScanIterationsRunner object at 0x7f198af26850>
@scan_chain_iterators: [<bliss.scanning.chain.AcquisitionChainIter object at 0x7f198a726910>, <bliss.scanning.chain.AcquisitionChainIter object at 0x7f198a726c70>]
@self: Scan(number=1, name=finterlaced, path=/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/L1_loading_f160N_bigbeam.h5)
File "/home/blissadm/local/bliss.git/bliss/scanning/scan.py", line 874, in prepare
self.writer.prepare(self)
@devices_tree: <treelib.tree.Tree object at 0x7f198d5de8b0>
@scan_info: ... (truncated)
@self: Scan(number=1, name=finterlaced, path=/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/L1_loading_f160N_bigbeam.h5)
File "/home/blissadm/local/bliss.git/bliss/scanning/writer/external_tango.py", line 93, in prepare
super().prepare(scan)
@__class__: <class 'bliss.scanning.writer.external_tango.ExternalTangoNexusWriter'>
@scan: Scan(number=1, name=finterlaced, path=/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/L1_loading_f160N_bigbeam.h5)
@self: <bliss.scanning.writer.external_tango.ExternalTangoNexusWriter object at 0x7f198a726e80>
File "/home/blissadm/local/bliss.git/bliss/scanning/writer/base.py", line 160, in prepare
self._prepare_device_saving(acq_master)
@acq_master: <bliss.scanning.acquisition.lima.LimaAcquisitionMaster object at 0x7f19b8cc9dc0>
@scan: Scan(number=1, name=finterlaced, path=/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/L1_loading_f160N_bigbeam.h5)
@self: <bliss.scanning.writer.external_tango.ExternalTangoNexusWriter object at 0x7f198a726e80>
File "/home/blissadm/local/bliss.git/bliss/scanning/writer/base.py", line 194, in _prepare_device_saving
acq_master.set_device_saving(directory, prefix)
@acq_master: <bliss.scanning.acquisition.lima.LimaAcquisitionMaster object at 0x7f19b8cc9dc0>
@device_root_path: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001/frelon3_'
@directory: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001'
@prefix: 'frelon3_'
@self: <bliss.scanning.writer.external_tango.ExternalTangoNexusWriter object at 0x7f198a726e80>
File "/home/blissadm/local/bliss.git/bliss/scanning/acquisition/lima.py", line 393, in set_device_saving
makedirs(self.acq_params["saving_directory"], exist_ok=True)
@directory: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001'
@force_no_saving: False
@prefix: 'frelon3_'
@self: <bliss.scanning.acquisition.lima.LimaAcquisitionMaster object at 0x7f19b8cc9dc0>
@validity: True
File "/home/blissadm/local/bliss.git/bliss/common/os_utils.py", line 21, in makedirs
if current_session.scan_saving.create_path(path):
@exist_ok: True
@kwargs: {}
@path: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001'
File "/home/blissadm/local/bliss.git/bliss/scanning/scan_saving.py", line 673, in create_path
return self.writer_object.create_path(path)
@path: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001'
@self: <bliss.config.settings.ESRFScanSaving object at 0x7f1a587ca640>
File "/home/blissadm/local/bliss.git/bliss/scanning/writer/external_tango.py", line 56, in create_path
self._proxy.makedirs(abspath)
@__class__: <class 'bliss.scanning.writer.external_tango.ExternalTangoNexusWriter'>
@abspath: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001'
@path: '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001'
@self: <bliss.scanning.writer.external_tango.ExternalTangoNexusWriter object at 0x7f1978cf8970>
File "/home/blissadm/local/bliss.git/bliss/common/tango.py", line 150, in logging_call
ret = tango_func(*args)
@args: ('/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001',)
@logger: <bound method Logger.debug of <BlissLogger id11/bliss_nxwriter/tdxrd (WARNING)>>
@name: 'makedirs'
@tango_func: <function __get_command_func.<locals>.f at 0x7f198a5b65e0>
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/device_proxy.py", line 279, in f
return dp.command_inout(name, *args, **kwds)
@args: ('/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001',)
@dp: NexusWriter(id11/bliss_nxwriter/tdxrd)
@kwds: {}
@name: 'makedirs'
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/green.py", line 195, in greener
return executor.run(fn, args, kwargs, wait=wait, timeout=timeout)
@access: <built-in method pop of dict object at 0x7f19833efd80>
@args: (NexusWriter(id11/bliss_nxwriter/tdxrd), 'makedirs', '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001')
@consume_green_mode: True
@executor: <tango.gevent_executor.GeventExecutor object at 0x7f1abda7fa60>
@fn: <function __Connection__command_inout at 0x7f19d48f7790>
@green_mode: None
@kwargs: {}
@obj: NexusWriter(id11/bliss_nxwriter/tdxrd)
@timeout: None
@wait: None
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/green.py", line 114, in run
return self.access(accessor, timeout=timeout)
@accessor: <gevent._gevent_cevent.AsyncResult object at 0x7f197884cdd0>
@args: (NexusWriter(id11/bliss_nxwriter/tdxrd), 'makedirs', '/data/visitor/ma6050/id11/20240416/RAW_DATA/L1_loading/L1_loading_f160N_bigbeam/scan0001')
@fn: <function __Connection__command_inout at 0x7f19d48f7790>
@kwargs: {}
@self: <tango.gevent_executor.GeventExecutor object at 0x7f1abda7fa60>
@timeout: None
@wait: True
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/gevent_executor.py", line 155, in access
return accessor.get(timeout=timeout)
@accessor: <gevent._gevent_cevent.AsyncResult object at 0x7f197884cdd0>
@self: <tango.gevent_executor.GeventExecutor object at 0x7f1abda7fa60>
@timeout: None
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/gevent_executor.py", line 68, in wrapper
return func(*args, **kwargs)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/connection.py", line 108, in __Connection__command_inout
r = Connection.command_inout_raw(self, name, *args, **kwds)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.9/site-packages/tango/connection.py", line 137, in __Connection__command_inout_raw
return self.__command_inout(cmd_name, param)
CommunicationFailed: DevFailed[
DevError[
desc = TRANSIENT CORBA system exception: TRANSIENT_CallTimedout
origin = Connection::command_inout()
reason = API_CorbaException
severity = ERR]
DevError[
desc = Timeout (3000 mS) exceeded on device id11/bliss_nxwriter/tdxrd, command makedirs
origin = Connection::command_inout()
reason = API_DeviceTimedOut
severity = ERR]
]
``