From e703a76402608a4d69b67cdc58277247573a34c1 Mon Sep 17 00:00:00 2001 From: Tobias Petersen Date: Fri, 17 Jul 2026 15:57:25 +0200 Subject: [PATCH 1/6] gh-119710: Let asyncio Process.wait() finish on only process exit. Letting Process.wait() only wait on actual process return is closer to how it's documented and consistent with Popen.wait(). This also reduces complexity for waking waiters which was inconsistend depending on ordering of wait/exit. --- Lib/asyncio/base_subprocess.py | 27 +++++------- Lib/test/test_asyncio/test_subprocess.py | 44 +++++++++++++++++-- ...-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst | 4 ++ 3 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 224b1883808a412..97db3fe12f17512 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,7 +26,6 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False - self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -214,7 +213,6 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) - self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -235,6 +233,16 @@ def _process_exited(self, returncode): if self._loop.get_debug(): logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode + + # gh-119710: Wake up futures waiting for wait() as soon as the process + # exits. The pipe transports now check for the loop being closed before + # scheduling a callback preventing gh-114177. This is consistent with + # the behavior prior to 3.11 and the documented semantics in _wait(). + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(returncode) + self._exit_waiters = None + if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen # object. On Python 3.6, it is required to avoid a ResourceWarning. @@ -258,15 +266,7 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return - if not self._pipes_connected: - # self._pipes_connected can be False if not all pipes were connected - # because either the process failed to start or the self._connect_pipes task - # got cancelled. In this broken state we consider all pipes disconnected and - # to avoid hanging forever in self._wait as otherwise _exit_waiters - # would never be woken up, we wake them up here. - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(self._returncode) + if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -276,11 +276,6 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(self._returncode) - self._exit_waiters = None self._loop = None self._proc = None self._protocol = None diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bb1a7d6ff22be8d..18ff6bac53a87d5 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -128,9 +128,6 @@ def test_proc_exited_no_invalid_state_error_on_exit_waiters(self): exit_waiter = self.loop.create_future() transport._exit_waiters.append(exit_waiter) - # _connect_pipes hasn't completed, so _pipes_connected is False. - self.assertFalse(transport._pipes_connected) - # Simulate process exit. _try_finish() will set the result on # exit_waiter because _pipes_connected is False, and then schedule # _call_connection_lost() because _pipes is empty (vacuously all @@ -436,6 +433,47 @@ async def len_message(message): self.assertEqual(output.rstrip(), b'3') self.assertEqual(exitcode, 0) + def test_wait_even_if_pipe_is_open(self): + # gh-119710: Process.wait() must return once the process exits even + # if its stdout pipe is inherited by a grandchild that keeps it open, + # so the pipe never reaches EOF. Otherwise wait() hangs forever + # despite the returncode being known. + + async def run(): + # Just setup a pipe to pass to the grandchild for reading to ensure it dies. + # Inheritable is to allow it to be passed on windows + r, w = os.pipe() + os.set_inheritable(r, True) + + code = textwrap.dedent(f"""\ + import subprocess, sys + subprocess.run([sys.executable, "-c", "import sys;sys.stdin.read()"]) + """) + + proc = await asyncio.create_subprocess_exec( + sys.executable, "-c", code, + # This will be inherited by granchild and should not prevent + # *this* process from firing .wait(). + stdout=subprocess.PIPE, + stdin=r, + pass_fds=(r,) if sys.platform != "win32" else (), + close_fds=False if sys.platform == "win32" else True, + ) + os.close(r) + + try: + # Ensure we start waiting before the process is killed. + wait_proc = asyncio.create_task(proc.wait()) + await asyncio.sleep(0.1) + proc.kill() + await asyncio.wait_for(wait_proc, timeout=2.0) + finally: + os.close(w) # Allows the grandchild to exit + if proc.stdout is not None: + await proc.stdout.read() + + self.loop.run_until_complete(run()) + def test_empty_input(self): async def empty_input(): diff --git a/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst b/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst new file mode 100644 index 000000000000000..45f99d72ea29a2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst @@ -0,0 +1,4 @@ +Fix :mod:`asyncio` subprocess :meth:`~asyncio.subprocess.Process.wait` +hanging when the process has exited but one of its pipes is kept open by an +inherited child process (so the pipe never reaches EOF). ``wait()`` now +returns as soon as the process exits, regardless of the pipes' state. From 13f7cc72bf92aeae7ae9f93990f432c763af5433 Mon Sep 17 00:00:00 2001 From: Tobias Alex-Petersen Date: Fri, 17 Jul 2026 16:07:41 +0200 Subject: [PATCH 2/6] Use sleep(0) in test per review comments. --- Lib/test/test_asyncio/test_subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 18ff6bac53a87d5..895907737495bb8 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -464,7 +464,7 @@ async def run(): try: # Ensure we start waiting before the process is killed. wait_proc = asyncio.create_task(proc.wait()) - await asyncio.sleep(0.1) + await asyncio.sleep(0) proc.kill() await asyncio.wait_for(wait_proc, timeout=2.0) finally: From 11103645bda91ff2fd74a618d975f2719d2b5c8f Mon Sep 17 00:00:00 2001 From: Tobias Alex-Petersen Date: Sun, 19 Jul 2026 11:46:15 +0200 Subject: [PATCH 3/6] Change to test_timeount to use support.SHORT_TIMEOUT --- Lib/test/test_asyncio/test_subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 895907737495bb8..5b3da87211ec4b9 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -466,7 +466,7 @@ async def run(): wait_proc = asyncio.create_task(proc.wait()) await asyncio.sleep(0) proc.kill() - await asyncio.wait_for(wait_proc, timeout=2.0) + await asyncio.wait_for(wait_proc, timeout=support.SHORT_TIMEOUT) finally: os.close(w) # Allows the grandchild to exit if proc.stdout is not None: From bc780d4d9683689bc2973579a6f9e80ac51987a5 Mon Sep 17 00:00:00 2001 From: Tobias Alex-Petersen Date: Sun, 19 Jul 2026 12:03:40 +0200 Subject: [PATCH 4/6] Remove internals heavy asyncio test that's covered elsewhere. --- Lib/test/test_asyncio/test_subprocess.py | 28 ------------------------ 1 file changed, 28 deletions(-) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 5b3da87211ec4b9..ebf27190081d5c1 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -112,34 +112,6 @@ def test_subprocess_repr(self): ) transport.close() - def test_proc_exited_no_invalid_state_error_on_exit_waiters(self): - # gh-145541: when _connect_pipes hasn't completed (so - # _pipes_connected is False) and the process exits, _try_finish() - # sets the result on exit waiters. Then _call_connection_lost() must - # not call set_result() again on the same waiters. - self.loop.set_exception_handler( - lambda loop, context: self.fail( - f"unexpected exception: {context}") - ) - waiter = self.loop.create_future() - transport, protocol = self.create_transport(waiter) - - # Simulate a waiter registered via _wait() before the process exits. - exit_waiter = self.loop.create_future() - transport._exit_waiters.append(exit_waiter) - - # Simulate process exit. _try_finish() will set the result on - # exit_waiter because _pipes_connected is False, and then schedule - # _call_connection_lost() because _pipes is empty (vacuously all - # disconnected). _call_connection_lost() must skip exit_waiter - # because it's already done. - transport._process_exited(6) - self.loop.run_until_complete(waiter) - - self.assertEqual(exit_waiter.result(), 6) - - transport.close() - class SubprocessMixin: From d8276f81167520ef00f1670dca00a11229a55fb2 Mon Sep 17 00:00:00 2001 From: Tobias Alex-Petersen Date: Sun, 19 Jul 2026 15:52:52 +0200 Subject: [PATCH 5/6] Move waiter wakeup to later in _process_exited This allows waiters to be scheduled after protocol.process_exited and/or connection_lost callbacks. --- Lib/asyncio/base_subprocess.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 97db3fe12f17512..3fa1ed1afd2a651 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -234,15 +234,6 @@ def _process_exited(self, returncode): logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode - # gh-119710: Wake up futures waiting for wait() as soon as the process - # exits. The pipe transports now check for the loop being closed before - # scheduling a callback preventing gh-114177. This is consistent with - # the behavior prior to 3.11 and the documented semantics in _wait(). - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(returncode) - self._exit_waiters = None - if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen # object. On Python 3.6, it is required to avoid a ResourceWarning. @@ -251,6 +242,13 @@ def _process_exited(self, returncode): self._try_finish() + # gh-119710: Wake up futures waiting for wait() as soon as the process + # exits. + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(returncode) + self._exit_waiters = None + async def _wait(self): """Wait until the process exit and return the process return code. From d723a091096d22f77f3f4bfb998f8ab4724ff8bd Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 19 Jul 2026 20:54:49 +0530 Subject: [PATCH 6/6] fix test --- Lib/test/test_asyncio/test_subprocess.py | 39 ++++++++++++------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index ebf27190081d5c1..2e97fa5820f8245 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -412,37 +412,36 @@ def test_wait_even_if_pipe_is_open(self): # despite the returncode being known. async def run(): - # Just setup a pipe to pass to the grandchild for reading to ensure it dies. - # Inheritable is to allow it to be passed on windows - r, w = os.pipe() - os.set_inheritable(r, True) - - code = textwrap.dedent(f"""\ + # The grandchild inherits the child's stdin and stdout pipes and + # keeps both open after the child is killed. It writes "ready" + # so we know it has started, and exits once its stdin hits EOF. + code = textwrap.dedent("""\ import subprocess, sys - subprocess.run([sys.executable, "-c", "import sys;sys.stdin.read()"]) + subprocess.run([sys.executable, "-c", + "import sys; sys.stdout.write('ready');" + " sys.stdout.flush(); sys.stdin.read()"]) """) proc = await asyncio.create_subprocess_exec( sys.executable, "-c", code, - # This will be inherited by granchild and should not prevent - # *this* process from firing .wait(). + stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stdin=r, - pass_fds=(r,) if sys.platform != "win32" else (), - close_fds=False if sys.platform == "win32" else True, ) - os.close(r) - try: - # Ensure we start waiting before the process is killed. wait_proc = asyncio.create_task(proc.wait()) - await asyncio.sleep(0) + # Wait until the grandchild holds the inherited pipes; this + # also lets the wait() task register its waiter. + await proc.stdout.readexactly(5) proc.kill() - await asyncio.wait_for(wait_proc, timeout=support.SHORT_TIMEOUT) + returncode = await asyncio.wait_for( + wait_proc, timeout=support.SHORT_TIMEOUT) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + else: + self.assertEqual(-signal.SIGKILL, returncode) finally: - os.close(w) # Allows the grandchild to exit - if proc.stdout is not None: - await proc.stdout.read() + proc.stdin.close() # let the grandchild exit + await proc.stdout.read() self.loop.run_until_complete(run())