large scans and oom
I ran a 25000 point scan over the weekend on id21 (500x500 amesh) using a simulated mca, and two real motors. Not clear to me how many points i got, but it fell over about 7 hours into the scan:
Traceback (most recent call last):
Traceback (most recent call last):
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3413, in _execute_transaction
response = self.parse_response(connection, '_')
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3479, in parse_response
self, connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/scanning/chain.py", line 338, in _start
return_value = self.start()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
File "/users/blissadm/local/bliss.git/bliss/scanning/acquisition/motor.py", line 397, in start
self.trigger()
redis.exceptions.ExecAbortError: Transaction discarded because of previous errors.
File "/users/blissadm/local/bliss.git/bliss/scanning/acquisition/motor.py", line 405, in trigger
self.channels.update_from_iterable(positions)
During handling of the above exception, another exception occurred:
File "/users/blissadm/local/bliss.git/bliss/scanning/channel.py", line 27, in update_from_iterable
channel.emit(iterable[i])
Traceback (most recent call last):
File "/users/blissadm/local/bliss.git/bliss/scanning/channel.py", line 135, in emit
dispatcher.send("new_data", self, data_dct)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/louie/dispatcher.py", line 365, in send
**named
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/louie/robustapply.py", line 65, in robust_apply
return receiver(*arguments, **named)
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 992, in _channel_event
self.nodes[sender].store(event_dict)
File "/users/blissadm/local/bliss.git/bliss/data/channel.py", line 94, in store
self.info.update(event_dict["description"])
File "/users/blissadm/local/bliss.git/bliss/data/scan.py", line 231, in _watch_data_callback
zerod_nodes, get_data_from_nodes(pipeline, *zerod_nodes_index)
File "/users/blissadm/local/bliss.git/bliss/config/settings.py", line 142, in _write
return func(self, values, **keys)
File "/users/blissadm/local/bliss.git/bliss/data/scan.py", line 129, in get_data_from_nodes
result = pipeline.execute()
File "/users/blissadm/local/bliss.git/bliss/config/settings.py", line 656, in update
cnx.hmset(self.name, values)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3520, in execute
return execute(conn, stack, raise_on_error)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 2693, in hmset
return self.execute_command('HMSET', name, *items)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3418, in _execute_transaction
raise errors[0][1]
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 775, in execute_command
return self.parse_response(connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3405, in _execute_transaction
self.parse_response(connection, '_')
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3479, in parse_response
self, connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
redis.exceptions.ResponseError: OOM command not allowed when used memory > 'maxmemory'.
redis.exceptions.ResponseError: Command # 1 (LRANGE session_sxm:data:id21:inhouse:19jul:bliss_tests:43_amesh:axis:timer:simul_mca:triggers_det0_data 50741 -1) of pipeline caused error: OOM command not allowed when used memory > 'maxmemory'.
2019-09-27T20:50:35Z <Greenlet at 0x7f5b649ad5f0: _watch_data_callback(<gevent._event.Event object at 0x7f5b62e321a0>, {<SCAN_EVENT.NEW_DATA: 3>: {'session_sxm:data:id21, <bound method Flint.new_scan of <bliss.flint.flint, <bound method Flint.new_scan_child of <bliss.flint, <bound method Flint.new_scan_data of <bliss.flint., <bound method Flint.end_scan of <bliss.flint.flint)> failed with ResponseError
Traceback (most recent call last):
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3413, in _execute_transaction
response = self.parse_response(connection, '_')
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3479, in parse_response
self, connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
redis.exceptions.ExecAbortError: Transaction discarded because of previous errors.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/data/scan.py", line 336, in watch_session_scans
for event_type, node in data_iterator.wait_for_event(pubsub):
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 385, in wait_for_event
channel_node = get_node(channel_db_name)
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 102, in get_node
return get_nodes(db_name, connection=connection)[0]
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 120, in get_nodes
for db_name, (name, node_type) in zip(db_names, grouped(pipeline.execute(), 2))
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3520, in execute
return execute(conn, stack, raise_on_error)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3418, in _execute_transaction
raise errors[0][1]
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3405, in _execute_transaction
self.parse_response(connection, '_')
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 3479, in parse_response
self, connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
redis.exceptions.ResponseError: Command # 1 (HGET session_sxm:data:id21:inhouse:19jul:bliss_tests:43_amesh:axis:timer:simul_mca:spectrum_det2 name) of pipeline caused error: OOM command not allowed when used memory > 'maxmemory'.
2019-09-27T20:50:35Z <Greenlet at 0x7f5b649ad4d0: watch_session_scans('session_sxm', <bound method Flint.new_scan of <bliss.flint.flint, <bound method Flint.new_scan_child of <bliss.flint, <bound method Flint.new_scan_data of <bliss.flint., <bound method Flint.end_scan of <bliss.flint.flint, ready_event=<gevent._event.Event object at 0x7f5b62e32360>)> failed with ResponseError
2019-09-27T20:50:35Z <Greenlet at 0x7ffaa5a4f830: <bound method AcquisitionMaster._start of <bliss.scanning.acquisition.motor.MeshStepTriggerMaster object at 0x7ffaa7a93750>>({'axis.wait_ready': [(1569591387.7189174, 15695913)> failed with ResponseError
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1222, in _run_next
i.start()
File "/users/blissadm/local/bliss.git/bliss/scanning/chain.py", line 721, in start
gevent.joinall(tasks, raise_error=True)
File "src/gevent/greenlet.py", line 899, in gevent._greenlet.joinall
File "src/gevent/greenlet.py", line 915, in gevent._greenlet.joinall
File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/scanning/chain.py", line 338, in _start
return_value = self.start()
File "/users/blissadm/local/bliss.git/bliss/scanning/acquisition/motor.py", line 397, in start
self.trigger()
File "/users/blissadm/local/bliss.git/bliss/scanning/acquisition/motor.py", line 405, in trigger
self.channels.update_from_iterable(positions)
File "/users/blissadm/local/bliss.git/bliss/scanning/channel.py", line 27, in update_from_iterable
channel.emit(iterable[i])
File "/users/blissadm/local/bliss.git/bliss/scanning/channel.py", line 135, in emit
dispatcher.send("new_data", self, data_dct)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/louie/dispatcher.py", line 365, in send
**named
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/louie/robustapply.py", line 65, in robust_apply
return receiver(*arguments, **named)
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 992, in _channel_event
self.nodes[sender].store(event_dict)
File "/users/blissadm/local/bliss.git/bliss/data/channel.py", line 94, in store
self.info.update(event_dict["description"])
File "/users/blissadm/local/bliss.git/bliss/config/settings.py", line 142, in _write
return func(self, values, **keys)
File "/users/blissadm/local/bliss.git/bliss/config/settings.py", line 656, in update
cnx.hmset(self.name, values)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 2693, in hmset
return self.execute_command('HMSET', name, *items)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 775, in execute_command
return self.parse_response(connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
redis.exceptions.ResponseError: OOM command not allowed when used memory > 'maxmemory'.
2019-09-27T20:50:36Z <Greenlet at 0x7ffa5506ddd0: <bound method Scan._run_next of Scan(number=43, name=amesh, path=/data/id21/inhouse/19jul/bliss_tests/session_sxm/data.h5)>(<bliss.scanning.chain.AcquisitionChainIter object )> failed with ResponseError
2019-09-27 22:50:36,651 bliss.scans DEBUG Start axis.stop
2019-09-27 22:50:36,651 global.controllers.iceid212.Command[iceid212:5000] DEBUG write bytes=11 b'STOP 42 43\n'
2019-09-27 22:50:36,651 bliss.scans DEBUG End axis.stop Took 0.000331s
2019-09-27 22:50:36,651 bliss.scans DEBUG Start timer.stop
2019-09-27 22:50:36,652 bliss.scans DEBUG End timer.stop Took 0.000036s
2019-09-27 22:50:36,652 bliss.scans DEBUG Start simul_mca.stop
2019-09-27 22:50:36,652 bliss.scans DEBUG Start timer.wait_slaves
2019-09-27 22:50:36,652 bliss.scans DEBUG End timer.wait_slaves Took 0.000257s
2019-09-27 22:50:36,653 bliss.scans DEBUG End timer.trigger Took 0.956045s
2019-09-27 22:50:36,754 bliss.scans DEBUG End simul_mca.stop Took 0.102078s
2019-09-27 22:50:36,754 bliss.scans DEBUG Start axis.wait_slaves
2019-09-27 22:50:36,754 bliss.scans DEBUG End axis.wait_slaves Took 0.000213s
2019-09-27 22:50:36,755 bliss.scans DEBUG Start timer.wait_slaves
2019-09-27 22:50:36,755 bliss.scans DEBUG End timer.wait_slaves Took 0.000075s
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/data/scan.py", line 377, in watch_session_scans
scan_dict = running_scans.pop(db_name)
KeyError: 'session_sxm:data:id21:inhouse:19jul:bliss_tests:43_amesh'
2019-09-27T20:50:36Z <Greenlet at 0x7f5b5be81a70: watch_session_scans('session_sxm', <bound method Flint.new_scan of <bliss.flint.flint, <bound method Flint.new_scan_child of <bliss.flint, <bound method Flint.new_scan_data of <bliss.flint., <bound method Flint.end_scan of <bliss.flint.flint, ready_event=<gevent._event.Event object at 0x7f5b62e32360>)> failed with KeyError
2019-09-27 22:50:37,357 user INFO Scan 43 finished
get_datacollections 5 {}
update_datacollection after {'sessionid': 1, 'datacollectionnumber': 43, 'sampleid': 1, 'subsampleid': 11, 'starttime': 1569591387.71817, 'experimenttype': 'zapimage', 'exposuretime': 0.1, 'steps_x': 500, 'steps_y': 500, 'datacollectionid': 5, 'endtime': 1569617437.3582218, 'status': 'failure'}
update_subsample 11 {'queued': 0}
queue_subsample after {'x': -6778306, 'type': 'roi', 'sampleid': 1, 'h': 500000, 'y': 2797189, 'w': 500000, 'subsampleid': 11, 'datacollections': 1, 'queued': 0}
2019-09-27 22:50:37,358 daiquiri.core.components DEBUG Actor failed roiscan 8c4f423a-e5eb-41d2-aa15-b2491772eb55 OOM command not allowed when used memory > 'maxmemory'.
2019-09-27 22:50:37,358 user ERROR Error running actor: roiscan 8c4f423a-e5eb-41d2-aa15-b2491772eb55, OOM command not allowed when used memory > 'maxmemory'.
Traceback (most recent call last):
File "/users/blissadm/local/daiquiri/id21/id21/backend/daiquiri/daiquiri/core/queue.py", line 312, in process
raise error
File "/users/blissadm/local/daiquiri/id21/id21/backend/daiquiri/daiquiri/core/components/__init__.py", line 406, in execute
self._resp = self.method(*self._args, **self._kwargs)
File "/users/blissadm/local/daiquiri/id21/id21/backend/daiquiri/implementors/id21/imageviewer/roiscan.py", line 95, in method
mesh.run()
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1157, in run
raise
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File "/users/blissadm/local/bliss.git/bliss/common/cleanup.py", line 276, in capture_exceptions
raise value
File "/users/blissadm/local/bliss.git/bliss/common/cleanup.py", line 252, in capture
yield
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1124, in run
gevent.joinall(wait_tasks, raise_error=True, count=1)
File "src/gevent/greenlet.py", line 899, in gevent._greenlet.joinall
File "src/gevent/greenlet.py", line 915, in gevent._greenlet.joinall
File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 1222, in _run_next
i.start()
File "/users/blissadm/local/bliss.git/bliss/scanning/chain.py", line 721, in start
gevent.joinall(tasks, raise_error=True)
File "src/gevent/greenlet.py", line 899, in gevent._greenlet.joinall
File "src/gevent/greenlet.py", line 915, in gevent._greenlet.joinall
File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/gevent/_compat.py", line 47, in reraise
raise value.with_traceback(tb)
File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/scanning/chain.py", line 338, in _start
return_value = self.start()
File "/users/blissadm/local/bliss.git/bliss/scanning/acquisition/motor.py", line 397, in start
self.trigger()
File "/users/blissadm/local/bliss.git/bliss/scanning/acquisition/motor.py", line 405, in trigger
self.channels.update_from_iterable(positions)
File "/users/blissadm/local/bliss.git/bliss/scanning/channel.py", line 27, in update_from_iterable
channel.emit(iterable[i])
File "/users/blissadm/local/bliss.git/bliss/scanning/channel.py", line 135, in emit
dispatcher.send("new_data", self, data_dct)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/louie/dispatcher.py", line 365, in send
**named
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/louie/robustapply.py", line 65, in robust_apply
return receiver(*arguments, **named)
File "/users/blissadm/local/bliss.git/bliss/scanning/scan.py", line 992, in _channel_event
self.nodes[sender].store(event_dict)
File "/users/blissadm/local/bliss.git/bliss/data/channel.py", line 94, in store
self.info.update(event_dict["description"])
File "/users/blissadm/local/bliss.git/bliss/config/settings.py", line 142, in _write
return func(self, values, **keys)
File "/users/blissadm/local/bliss.git/bliss/config/settings.py", line 656, in update
cnx.hmset(self.name, values)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 2693, in hmset
return self.execute_command('HMSET', name, *items)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 775, in execute_command
return self.parse_response(connection, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 789, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 642, in read_response
raise response
redis.exceptions.ResponseError: OOM command not allowed when used memory > 'maxmemory'.
2019-09-27 22:50:37,362 daiquiri.core.queue DEBUG Queued actor failed, stopping queue
2019-09-27 22:50:37,413 daiquiri.core DEBUG Core:before_request GET
2019-09-27 22:50:37,413 daiquiri.core.session DEBUG Session: require_valid_session
2019-09-27 22:50:43,374 daiquiri.core DEBUG Core:before_request GET
2019-09-27 22:50:43,375 daiquiri.core.session DEBUG Session: require_valid_session
I'll do some further testing, but its not clear to me if this is
- bliss
- because i'm watching the session (session_watch_scan)
- have a reference somewhere that should have been garbage collected
I dont store the results of the scan in in memory, only fetch them when needed.
Edited by Stuart Fisher