From fa320570354f9eef7efcd3f6e719cfe38f124139 Mon Sep 17 00:00:00 2001 From: "Fabian @ Blax Software" Date: Thu, 2 Jul 2026 11:06:54 +0200 Subject: [PATCH] fix(server): reliably reap fork-per-message child processes (zombie leak) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fork-per-message model spawns a child per inbound message; each child exit(0)s when done. The only reaper was a single, non-looping pcntl_waitpid(-1, WNOHANG) inside the IPC onClose callback — which fires on socket EOF, an event that routinely precedes the child's actual process exit, so it frequently reaped nothing and left zombies parented to the live serve process (where no PID 1 init can reach them). Under sustained traffic these accumulated until pcntl_fork() itself failed. - Handler::onClose now drains ALL exited children in a loop (was a single wait). activeChildCount bookkeeping stays per-socket, untouched. - StartServer adds a SIGCHLD handler + a 10s periodic backstop (catches children that crashed before IPC setup, so no onClose fires) + a drain-on-shutdown so a deploy restart does not hand orphans to PID 1. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/Console/Commands/StartServer.php | 83 ++++++++++++++++++++++++++++ src/Websocket/Handler.php | 16 +++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/src/Console/Commands/StartServer.php b/src/Console/Commands/StartServer.php index 3dd58b7..868a3b5 100644 --- a/src/Console/Commands/StartServer.php +++ b/src/Console/Commands/StartServer.php @@ -177,6 +177,10 @@ class StartServer extends Command $this->configurePcntlSignal(); \Log::channel('websocket')->debug('PCNTL signals configured'); + \Log::channel('websocket')->debug('Configuring zombie reaper...'); + $this->configureZombieReaper(); + \Log::channel('websocket')->debug('Zombie reaper configured'); + \Log::channel('websocket')->debug('Configuring broadcast socket...'); $this->configureBroadcastSocket(); \Log::channel('websocket')->debug('Broadcast socket configured'); @@ -478,6 +482,76 @@ class StartServer extends Command \Log::channel('websocket')->debug('SIGINT handler registered'); } + /** + * Reap any exited fork-per-message child processes. + * + * The fork-per-message model (Handler::forkWithSocketPair) spawns a child + * per inbound message; each child exit(0)s when it is done. If the parent + * never waitpid()s them, they linger as zombies that consume PID + * table slots until pcntl_fork() itself fails. This drains ALL currently + * exited children in one non-blocking sweep (WNOHANG never blocks), so it is + * safe to call from a SIGCHLD handler, a periodic timer, or the shutdown + * path. It only ever reaps children that have already exited. + * + * @return int number of children reaped this sweep + */ + protected function reapChildren(): int + { + if (! function_exists('pcntl_waitpid')) { + return 0; + } + + $reaped = 0; + while (($pid = pcntl_waitpid(-1, $status, WNOHANG)) > 0) { + $reaped++; + } + + return $reaped; + } + + /** + * Install the zombie reaper for fork-per-message children. + * + * This is independent of, and a superset of, the IPC onClose reaper in + * Handler: onClose is keyed on SOCKET close, which routinely fires *before* + * the child's process actually exits, so it frequently reaps nothing and + * leaves the child behind as a zombie parented to this (live) serve process + * — where no PID 1 init can ever reach it. Two belt-and-suspenders reapers: + * + * - SIGCHLD handler → prompt reaping the instant any child changes state. + * Registered through the loop for consistency with the SIGTERM/SIGINT + * handlers; the WNOHANG drain loop makes signal coalescing harmless. + * - Periodic timer → guaranteed backstop that also catches children which + * crashed before their IPC socket was set up (no onClose ever fires) and + * runs even during idle periods with no socket activity. + * + * @return void + */ + protected function configureZombieReaper(): void + { + if (! function_exists('pcntl_waitpid')) { + \Log::channel('websocket')->warning('pcntl_waitpid unavailable — fork children cannot be reaped; zombies may accumulate.'); + return; + } + + // Prompt reaping: SIGCHLD fires as soon as a forked child exits. + $this->loop->addSignal(SIGCHLD, function () { + $this->reapChildren(); + }); + \Log::channel('websocket')->debug('SIGCHLD reaper registered'); + + // Guaranteed backstop: drains stragglers (e.g. a child that died before + // setupChild() so no onClose fires, or any child missed by SIGCHLD + // coalescing) even when there is no socket activity to trigger onClose. + $this->loop->addPeriodicTimer(10, function () { + $reaped = $this->reapChildren(); + if ($reaped > 0) { + \Log::channel('websocket')->debug('Periodic reaper collected ' . $reaped . ' zombie child process(es)'); + } + }); + \Log::channel('websocket')->debug('Periodic zombie reaper registered (10s)'); + } + /** * Configure the broadcast socket server for efficient broadcasting. * @@ -869,6 +943,11 @@ class StartServer extends Command // Stop the broadcast socket server $this->stopBroadcastSocket(); + // Reap any already-exited fork children before we exit so they are not + // handed to PID 1 as orphans (still-running children are unavoidably + // reparented and reaped by the container init/supervisord). + $this->reapChildren(); + $this->loop->stop(); } @@ -909,6 +988,10 @@ class StartServer extends Command // Stop the broadcast socket server $this->stopBroadcastSocket(); + // Reap any already-exited fork children before we exit (see + // triggerHardShutdown). + $this->reapChildren(); + $this->loop->stop(); }); } diff --git a/src/Websocket/Handler.php b/src/Websocket/Handler.php index 39e81f3..f24253d 100644 --- a/src/Websocket/Handler.php +++ b/src/Websocket/Handler.php @@ -739,10 +739,20 @@ class Handler implements MessageComponentInterface // Same isolation rules apply: must not throw out of the loop. function () { try { - // Cleanup zombie process - pcntl_waitpid(-1, $status, WNOHANG); + // Reap ALL exited fork children, not just one. onClose fires on + // socket EOF, which can precede the child's actual process exit — + // so this call often reaps the *previous* message's child (or + // nothing yet) rather than this one. Draining in a loop keeps + // zombies from accumulating between messages; StartServer's + // SIGCHLD + periodic reaper is the idle/straggler backstop. + while (pcntl_waitpid(-1, $status, WNOHANG) > 0) { + // reaped one child + } - // Free up a child slot and process any queued messages + // Free up a child slot and process any queued messages. + // NOTE: this decrement is per-child-SOCKET (one onClose per + // child) and must stay here — do NOT tie it to the reap count + // above, which reaps an arbitrary number of unrelated children. $this->activeChildCount = max(0, $this->activeChildCount - 1); $this->processDeferredMessages(); } catch (\Throwable $e) {