Nexus writer: File descriptor was closed in another greenlet
Reported by @debionne at ID13
ERROR 2023-03-05 13:23:11,362 nexus_writer_service.subscribers.session_writer: [eh2-0 (FAULT)] Error while reading from socket: (9, 'File descriptor was closed in another greenlet')
ERROR 2023-03-05 13:23:11,362 nexus_writer_service.subscribers.session_writer: [eh2-0 (FAULT)] Stop listening to Redis events due to exception
Traceback (most recent call last):
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 421, in read_from_socket
bufflen = recv_into(self._sock, self._buffer)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/_compat.py", line 75, in recv_into
return sock.recv_into(*args, **kwargs)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/gevent/_socketcommon.py", line 693, in recv_into
self._wait(self._read_event)
File "src/gevent/_hub_primitives.py", line 317, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src/gevent/_hub_primitives.py", line 322, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src/gevent/_hub_primitives.py", line 304, in gevent._gevent_c_hub_primitives._primitive_wait
File "src/gevent/_hub_primitives.py", line 46, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 46, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 55, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_waiter.py", line 154, in gevent._gevent_c_waiter.Waiter.get
File "src/gevent/_greenlet_primitives.py", line 61, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 61, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 65, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_gevent_c_greenlet_primitives.pxd", line 35, in gevent._gevent_c_greenlet_primitives._greenlet_switch
gevent._socketcommon.cancel_wait_ex: [Errno 9] File descriptor was closed in another greenlet
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/base_subscriber.py", line 389, in __greenlet_main
self._listen_event_loop()
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/base_subscriber.py", line 441, in _listen_event_loop
stop_handler=self._stop_handler, started_event=self._started_event
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/session_writer.py", line 178, in _walk_events
yield from self.node.walk_on_new_events(**kwargs)
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 1132, in walk_on_new_events
yield from self.walk_events(first_index=streaming.DataStream.now_index(), **kw)
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 1123, in walk_events
started_event=started_event,
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 1177, in _iter_reader
for stream, events in reader:
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 809, in __iter__
raise item
File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 687, in _read_task_main
lst = self._read_active_streams()
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 629, in _read_active_streams
streams_to_read, count=self._count, block=self._block
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 2573, in xread
return self.execute_command('XREAD', *pieces)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 901, in execute_command
return self.parse_response(conn, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 915, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 730, in read_response
response = self._parser.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 464, in read_response
self.read_from_socket()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 447, in read_from_socket
(ex.args,))
redis.exceptions.ConnectionError: Error while reading from socket: (9, 'File descriptor was closed in another greenlet')
Traceback (most recent call last):
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 421, in read_from_socket
bufflen = recv_into(self._sock, self._buffer)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/_compat.py", line 75, in recv_into
return sock.recv_into(*args, **kwargs)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/gevent/_socketcommon.py", line 693, in recv_into
self._wait(self._read_event)
File "src/gevent/_hub_primitives.py", line 317, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src/gevent/_hub_primitives.py", line 322, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src/gevent/_hub_primitives.py", line 304, in gevent._gevent_c_hub_primitives._primitive_wait
File "src/gevent/_hub_primitives.py", line 46, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 46, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 55, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_waiter.py", line 154, in gevent._gevent_c_waiter.Waiter.get
File "src/gevent/_greenlet_primitives.py", line 61, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 61, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 65, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_gevent_c_greenlet_primitives.pxd", line 35, in gevent._gevent_c_greenlet_primitives._greenlet_switch
gevent._socketcommon.cancel_wait_ex: [Errno 9] File descriptor was closed in another greenlet
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 687, in _read_task_main
lst = self._read_active_streams()
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 629, in _read_active_streams
streams_to_read, count=self._count, block=self._block
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 2573, in xread
return self.execute_command('XREAD', *pieces)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 901, in execute_command
return self.parse_response(conn, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 915, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 730, in read_response
response = self._parser.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 464, in read_response
self.read_from_socket()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 447, in read_from_socket
(ex.args,))
redis.exceptions.ConnectionError: Error while reading from socket: (9, 'File descriptor was closed in another greenlet')
2023-03-05T12:23:11Z <BlissGreenlet at 0x7f1363bb0b90: <bound method DataStreamReader._read_task_main of <bliss.config.streaming.DataStreamReader object at 0x7f1363b620d0>>> failed with ConnectionError
Traceback (most recent call last):
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 421, in read_from_socket
bufflen = recv_into(self._sock, self._buffer)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/_compat.py", line 75, in recv_into
return sock.recv_into(*args, **kwargs)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/gevent/_socketcommon.py", line 693, in recv_into
self._wait(self._read_event)
File "src/gevent/_hub_primitives.py", line 317, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src/gevent/_hub_primitives.py", line 322, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src/gevent/_hub_primitives.py", line 304, in gevent._gevent_c_hub_primitives._primitive_wait
File "src/gevent/_hub_primitives.py", line 46, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 46, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_hub_primitives.py", line 55, in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait
File "src/gevent/_waiter.py", line 154, in gevent._gevent_c_waiter.Waiter.get
File "src/gevent/_greenlet_primitives.py", line 61, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 61, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_greenlet_primitives.py", line 65, in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch
File "src/gevent/_gevent_c_greenlet_primitives.pxd", line 35, in gevent._gevent_c_greenlet_primitives._greenlet_switch
gevent._socketcommon.cancel_wait_ex: [Errno 9] File descriptor was closed in another greenlet
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/base_subscriber.py", line 352, in _greenlet_main
self.__greenlet_main()
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/base_subscriber.py", line 389, in __greenlet_main
self._listen_event_loop()
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/base_subscriber.py", line 441, in _listen_event_loop
stop_handler=self._stop_handler, started_event=self._started_event
File "/users/blissadm/local/bliss.git/nexus_writer_service/subscribers/session_writer.py", line 178, in _walk_events
yield from self.node.walk_on_new_events(**kwargs)
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 1132, in walk_on_new_events
yield from self.walk_events(first_index=streaming.DataStream.now_index(), **kw)
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 1123, in walk_events
started_event=started_event,
File "/users/blissadm/local/bliss.git/bliss/data/node.py", line 1177, in _iter_reader
for stream, events in reader:
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 809, in __iter__
raise item
File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 687, in _read_task_main
lst = self._read_active_streams()
File "/users/blissadm/local/bliss.git/bliss/config/streaming.py", line 629, in _read_active_streams
streams_to_read, count=self._count, block=self._block
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 2573, in xread
return self.execute_command('XREAD', *pieces)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 901, in execute_command
return self.parse_response(conn, command_name, **options)
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/client.py", line 915, in parse_response
response = connection.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 730, in read_response
response = self._parser.read_response()
File "/users/blissadm/conda/miniconda/envs/bliss_dev/lib/python3.7/site-packages/redis/connection.py", line 464, in read_response
self.read_from_socket()