Test failures on hydra look suspiciously like they stem from xdist.
gw0 ok / gw1 ok / gw2 ok / gw3 ok / gw4 ok / gw5 ok / gw6 ok / gw7 ok / gw8 ok / gw9 ok / gw10 ok / gw11 ok / gw12 ok / gw13 ok / gw14 ok / gw15 ok / gw16 ok / gw17 ok / gw18 ok / gw19 C / gw20 C / gw21 C / gw22 C / gw23 C / gw24 C / gw25 C / gw26 C / gw27 C / gw28 C / gw29 C / gw30 C / gw31 C / gw32 C / gw33 C / gw34 C / gw35 C / gw36 C / gw37 C / gw38 C / gw39 C / gw40 C / gw41 C / gw42 C / gw43 C / gw44 C / gw45 C / gw46 C / gw47 CINTERNALERROR> def worker_internal_error(self, node, formatted_error):
INTERNALERROR> """
INTERNALERROR> pytest_internalerror() was called on the worker.
INTERNALERROR>
INTERNALERROR> pytest_internalerror() arguments are an excinfo and an excrepr, which can't
INTERNALERROR> be serialized, so we go with a poor man's solution of raising an exception
INTERNALERROR> here ourselves using the formatted message.
INTERNALERROR> """
INTERNALERROR> self._active_nodes.remove(node)
INTERNALERROR> try:
INTERNALERROR> > assert False, formatted_error
INTERNALERROR> E AssertionError: Traceback (most recent call last):
INTERNALERROR> E File "/nix/store/4v6lhz8sq9jwl3af8abs6cgpbh9lignr-python3.8-pytest-6.2.3/lib/python3.8/site-packages/_pytest/main.py", line 267, in wrap_session
INTERNALERROR> E config.hook.pytest_sessionstart(session=session)
INTERNALERROR> E File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/hooks.py", line 286, in __call__
INTERNALERROR> E return self._hookexec(self, self.get_hookimpls(), kwargs)
INTERNALERROR> E File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/manager.py", line 93, in _hookexec
INTERNALERROR> E return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR> E File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/manager.py", line 84, in <lambda>
INTERNALERROR> E self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(
INTERNALERROR> E File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/callers.py", line 208, in _multicall
INTERNALERROR> E return outcome.get_result()
INTERNALERROR> E File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/callers.py", line 80, in get_result
INTERNALERROR> E raise ex[1].with_traceback(ex[2])
INTERNALERROR> E File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/callers.py", line 187, in _multicall
INTERNALERROR> E res = hook_impl.function(*args)
INTERNALERROR> E File "/build/SQLAlchemy-1.4.15/test/../lib/sqlalchemy/testing/plugin/pytestplugin.py", line 135, in pytest_sessionstart
INTERNALERROR> E asyncio._assume_async(plugin_base.post_begin)
INTERNALERROR> E File "/nix/store/03hwa29jf7794x48983j44g0qvancijw-python3.8-SQLAlchemy-1.4.15/lib/python3.8/site-packages/sqlalchemy/testing/asyncio.py", line 50, in _assume_async
INTERNALERROR> E return _util_async_run(fn, *args, **kwargs)
INTERNALERROR> E File "/nix/store/03hwa29jf7794x48983j44g0qvancijw-python3.8-SQLAlchemy-1.4.15/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 167, in _util_async_run
INTERNALERROR> E loop = asyncio.get_event_loop()
INTERNALERROR> E File "/nix/store/4s0h5aawbap3xhldxhcijvl26751qrjr-python3-3.8.9/lib/python3.8/asyncio/events.py", line 639, in get_event_loop
INTERNALERROR> E raise RuntimeError('There is no current event loop in thread %r.'
INTERNALERROR> E RuntimeError: There is no current event loop in thread 'Dummy-1'.
INTERNALERROR> E assert False
INTERNALERROR>
INTERNALERROR> /nix/store/ws9fk09ssyzhy93i8janlh274nay3190-python3.8-pytest-xdist-2.2.1/lib/python3.8/site-packages/xdist/dsession.py:187: AssertionError
[gw18] node down: Not properly terminated
replacing crashed worker gw18
[gw48] linux Python 3.8.9 cwd: /build/SQLAlchemy-1.4.15
gw0 ok / gw1 ok / gw2 ok / gw3 ok / gw4 ok / gw5 ok / gw6 ok / gw7 ok / gw8 ok / gw9 ok / gw10 ok / gw11 ok / gw12 ok / gw13 ok / gw14 ok / gw15 ok / gw16 ok / gw17 ok / gw48 C / gw19 C / gw20 C / gw21 C / gw22 C / gw23 C / gw24 C / gw25 C / gw26 C / gw27 C / gw28 C / gw29 C / gw30 C / gw31 C / gw32 C / gw33 C / gw34 C / gw35 C / gw36 C / gw37 C / gw38 C / gw39 C / gw40 C / gw41 C / gw42 C / gw43 C / gw44 C / gw45 C / gw46 C / gw47 CINTERNALERROR> Traceback (most recent call last):
INTERNALERROR> File "/nix/store/4v6lhz8sq9jwl3af8abs6cgpbh9lignr-python3.8-pytest-6.2.3/lib/python3.8/site-packages/_pytest/main.py", line 269, in wrap_session
INTERNALERROR> session.exitstatus = doit(config, session) or 0
INTERNALERROR> File "/nix/store/4v6lhz8sq9jwl3af8abs6cgpbh9lignr-python3.8-pytest-6.2.3/lib/python3.8/site-packages/_pytest/main.py", line 323, in _main
INTERNALERROR> config.hook.pytest_runtestloop(session=session)
INTERNALERROR> File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/hooks.py", line 286, in __call__
INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs)
INTERNALERROR> File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/manager.py", line 93, in _hookexec
INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR> File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/manager.py", line 84, in <lambda>
INTERNALERROR> self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(
INTERNALERROR> File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/callers.py", line 208, in _multicall
INTERNALERROR> return outcome.get_result()
INTERNALERROR> File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/callers.py", line 80, in get_result
INTERNALERROR> raise ex[1].with_traceback(ex[2])
INTERNALERROR> File "/nix/store/r4lwmmknxwx3gq2bv73yf0rkli9d902d-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/callers.py", line 187, in _multicall
INTERNALERROR> res = hook_impl.function(*args)
INTERNALERROR> File "/nix/store/ws9fk09ssyzhy93i8janlh274nay3190-python3.8-pytest-xdist-2.2.1/lib/python3.8/site-packages/xdist/dsession.py", line 112, in pytest_runtestloop
INTERNALERROR> self.loop_once()
INTERNALERROR> File "/nix/store/ws9fk09ssyzhy93i8janlh274nay3190-python3.8-pytest-xdist-2.2.1/lib/python3.8/site-packages/xdist/dsession.py", line 135, in loop_once
INTERNALERROR> call(**kwargs)
INTERNALERROR> File "/nix/store/ws9fk09ssyzhy93i8janlh274nay3190-python3.8-pytest-xdist-2.2.1/lib/python3.8/site-packages/xdist/dsession.py", line 224, in worker_errordown
INTERNALERROR> self._active_nodes.remove(node)
INTERNALERROR> KeyError: <WorkerController gw18>
77673388f5