I get "anyio._backends._asyncio.ExceptionGroup: 0 exceptions were raised in the task group"

The following code reproduces the issue. Sorry about the length of it.

from __future__ import annotations

from contextlib import suppress
from pathlib import Path
from subprocess import STDOUT, CalledProcessError
from typing import Optional, Sequence, Union

import anyio
from anyio.abc import Process
from anyio.streams.file import FileReadStream
from anyio.streams.text import TextReceiveStream


async def run_process(
    # tg: TaskGroup,
    command: Union[str, Sequence[str]],
    *,
    input_for_stdin: Optional[Path] = None,
    raise_on_rc: Optional[bool] = None,
) -> None:
    """Run the given command as a foreground process.

    Unlike `anyio.run_process`, this streams data to/from the process while the
    process runs. This way, you can see the process' output while it's running.
    Useful for long-running processes.
    """
    process: Optional[Process] = None
    try:
        process = await anyio.open_process(command, stderr=STDOUT)
        await drain_streams(process, input_for_stdin)
    except BaseException:
        if process is not None:
            # Try to gracefully terminate the process
            process.terminate()
            # Give the process some time to stop
            with anyio.move_on_after(5, shield=True):
                await drain_streams(process)
        raise
    finally:
        if process is not None:
            # We tried to be graceful. Now there is no mercy.
            with suppress(ProcessLookupError):
                process.kill()
            # Close the streams (stdin, stdout, stderr)
            await process.aclose()

    assert process.returncode is not None
    # Check the return code (rc)
    if raise_on_rc and process.returncode != 0:
        raise CalledProcessError(process.returncode, command)


async def drain_streams(
    process: Process, input_for_stdin: Optional[Path] = None
) -> None:
    async with anyio.create_task_group() as tg:
        # In parallel:
        #  * send to stdin
        #  * receive from stdout
        if process.stdin is not None and input_for_stdin is not None:
            tg.start_soon(_send_to_stdin, process, input_for_stdin)
        if process.stdout is not None:
            tg.start_soon(_receive_from_stdout, process)
        # Wait for normal exit
        await process.wait()


async def _send_to_stdin(process: Process, input_for_stdin: Path) -> None:
    assert process.stdin is not None
    # Forward data from file to stdin
    async with await FileReadStream.from_path(input_for_stdin) as chunks:
        async for chunk in chunks:
            await process.stdin.send(chunk)


async def _receive_from_stdout(process: Process) -> None:
    assert process.stdout is not None
    # Forward data from stdout
    async for string in TextReceiveStream(process.stdout):
        print(string)


async def main():
    async with anyio.create_task_group() as tg:
        # Run the process in the "background"
        tg.start_soon(run_process, ("sleep", "10"))
        # We can do something else while the process runs
        print("Sleeping now. Try to press CTRL+C.")
        await anyio.sleep(10)


anyio.run(main)

Try to press CTRL+C while it runs. Example stack trace:

Sleeping now. Try to press CTRL+C.
^Cunhandled exception during asyncio.run() shutdown
task: <Task finished name='__main__.run_process' coro=<run_process() done, defined at /projects/stork/anyio_bug.py:14> exception=<ExceptionGroup: >>
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 629, in run_until_complete
    self.run_forever()
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 596, in run_forever
    self._run_once()
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 1854, in _run_once
    event_list = self._selector.select(timeout)
  File "/usr/local/lib/python3.9/selectors.py", line 469, in select
    fd_event_list = self._selector.poll(timeout, max_ev)
KeyboardInterrupt

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/projects/stork/anyio_bug.py", line 65, in drain_streams
    await process.wait()
  File "/projects/stork/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 825, in wait
    return await self._process.wait()
  File "/usr/local/lib/python3.9/asyncio/subprocess.py", line 135, in wait
    return await self._transport._wait()
  File "/usr/local/lib/python3.9/asyncio/base_subprocess.py", line 235, in _wait
    return await waiter
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/projects/stork/anyio_bug.py", line 30, in run_process
    await drain_streams(process, input_for_stdin)
  File "/projects/stork/anyio_bug.py", line 65, in drain_streams
    await process.wait()
  File "/projects/stork/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 526, in __aexit__
    raise ExceptionGroup(exceptions)
anyio._backends._asyncio.ExceptionGroup: 0 exceptions were raised in the task group:
----------------------------

Traceback (most recent call last):
  File "/projects/stork/anyio_bug.py", line 92, in <module>
    anyio.run(main)
  File "/projects/stork/.venv/lib/python3.9/site-packages/anyio/_core/_eventloop.py", line 55, in run
    return asynclib.run(func, *args, **backend_options)  # type: ignore
  File "/projects/stork/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 211, in run
    return native_run(wrapper(), debug=debug)
  File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 629, in run_until_complete
    self.run_forever()
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 596, in run_forever
    self._run_once()
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 1854, in _run_once
    event_list = self._selector.select(timeout)
  File "/usr/local/lib/python3.9/selectors.py", line 469, in select
    fd_event_list = self._selector.poll(timeout, max_ev)
KeyboardInterrupt

The interesting part to me is the anyio._backends._asyncio.ExceptionGroup: 0 exceptions were raised in the task group message. Why does anyio raise an ExceptionGroup with zero exceptions in it?

If I’m doing something that I’m not supposed to, let me know. 😃

[graingert]

I think that sounds like a good and pragmatic solution.

Does this fix:

* avoid the whole `unhandled exception during asyncio.run() shutdown` situation to begin with?

yes, asyncio.run ignores CancelledError exceptions here https://github.com/python/cpython/blob/ce47addfb6f176fad053431b537b77a5f170765e/Lib/asyncio/runners.py#L67-L68

* simply change the stack trace from  `anyio._backends._asyncio.ExceptionGroup: 0 exceptions were raised in the task group` to `asyncio.exceptions.CancelledError`?

there won’t be a an ExceptionGroup as long as all the exceptions are CancelledError

I’d prefer the former but maybe that’s impossible with the asyncio backend.

[agronholm]

The line where the exceptions list is replaced can be found here: https://github.com/agronholm/anyio/blob/master/src/anyio/_backends/_asyncio.py#L559

It’s not clear if the logic for when to filter out CancelledError is correct. Perhaps we should be checking if the host task has been cancelled. Either way, I believe I can fix this one particular wart without any side effects.