scan groups and caching race condition
Stress testing revealed the exception below in GroupingMaster._new_subscan
.
- Each scans create their own
CachingRedisConnectionPool
with their ownRedisCache
- After the scan the
RedisCache
is disabled so all connections behave like normal uncached connection -
GroupingMaster._new_subscan
sets the ttl of the scan node and its children
I'm not sure yet when GroupingMaster._new_subscan
gets called but there must be a race condition with disable_caching
in the sub scans.
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 854, in gevent._gevent_cgreenlet.Greenlet.run
File "/home/denolf/dev/bliss/bliss/config/streaming.py", line 669, in _read_task_main
lst = self._read_active_streams()
File "/home/denolf/dev/bliss/bliss/config/streaming.py", line 610, in _read_active_streams
streams_to_read, count=self._count, block=self._block
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/client.py", line 2573, in xread
return self.execute_command('XREAD', *pieces)
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/client.py", line 898, in execute_command
conn = self.connection or pool.get_connection(command_name, **options)
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/connection.py", line 1182, in get_connection
connection.connect()
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/connection.py", line 567, in connect
callback(self)
File "/home/denolf/dev/bliss/bliss/config/conductor/redis_caching.py", line 269, in track_connection
self._invalidation_greenlet.track_connection(connection)
File "/home/denolf/dev/bliss/bliss/config/conductor/redis_caching.py", line 160, in track_connection
raise RedisCacheError("Redis key invalidation greenlet is not running")
bliss.config.conductor.redis_caching.RedisCacheError: Redis key invalidation greenlet is not running
2020-12-13T14:59:40Z <Greenlet at 0x7f7c3ea6d3b0: <bound method DataStreamReader._read_task_main of <bliss.config.streaming.DataStreamReader object at 0x7f7c3eab3e90>>> failed with RedisCacheError
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 854, in gevent._gevent_cgreenlet.Greenlet.run
File "/home/denolf/dev/bliss/bliss/scanning/scan.py", line 489, in _run_next
for i in next_iter:
File "/home/denolf/dev/bliss/bliss/scanning/chain.py", line 983, in __next__
next(acq_obj_iter)
File "/home/denolf/dev/bliss/bliss/scanning/chain.py", line 152, in __next__
self.__current_acq_object = next(self.__iterator)
File "/home/denolf/dev/bliss/bliss/scanning/group.py", line 306, in __iter__
self._new_subscan(scan)
File "/home/denolf/dev/bliss/bliss/scanning/group.py", line 320, in _new_subscan
for n in scan.walk(wait=False):
File "/home/denolf/dev/bliss/bliss/data/node.py", line 911, in walk
started_event=started_event,
File "/home/denolf/dev/bliss/bliss/data/node.py", line 1071, in _iter_reader
for stream, events in reader:
File "/home/denolf/dev/bliss/bliss/config/streaming.py", line 776, in __iter__
raise item
File "src/gevent/greenlet.py", line 854, in gevent._gevent_cgreenlet.Greenlet.run
File "/home/denolf/dev/bliss/bliss/config/streaming.py", line 669, in _read_task_main
lst = self._read_active_streams()
File "/home/denolf/dev/bliss/bliss/config/streaming.py", line 610, in _read_active_streams
streams_to_read, count=self._count, block=self._block
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/client.py", line 2573, in xread
return self.execute_command('XREAD', *pieces)
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/client.py", line 898, in execute_command
conn = self.connection or pool.get_connection(command_name, **options)
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/connection.py", line 1182, in get_connection
connection.connect()
File "/users/denolf/virtualenvs/bliss_env/lib/python3.7/site-packages/redis/connection.py", line 567, in connect
callback(self)
File "/home/denolf/dev/bliss/bliss/config/conductor/redis_caching.py", line 269, in track_connection
self._invalidation_greenlet.track_connection(connection)
File "/home/denolf/dev/bliss/bliss/config/conductor/redis_caching.py", line 160, in track_connection
raise RedisCacheError("Redis key invalidation greenlet is not running")
bliss.config.conductor.redis_caching.RedisCacheError: ('GroupingMaster', 'Redis key invalidation greenlet is not running')
2020-12-13T14:59:40Z <Greenlet at 0x7f7c3f599dd0: <bound method _ScanIterationsRunner._run_next of <bliss.scanning.scan._ScanIterationsRunner object at 0x7f7c3d18b1d0>>(Scan(number=4014, name=sequence_of_scans, path=/da, <bliss.scanning.chain.AcquisitionChainIter object )> failed with RedisCacheError