fix(server): retry port bind on EADDRINUSE during restart handover

A restarting daemon (deploy, supervisor, steer restart-hard) often finds
the predecessor's socket still bound for a few seconds after SIGTERM.
Crashing immediately turned every restart into an EADDRINUSE crash-loop
(146 GlitchTip events on learn-atc prod). Retry the bind for up to 30s
before giving up; a port still taken after that is a real conflict and
rethrows.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Fabian @ Blax Software 2026-07-17 08:41:50 +02:00
parent fa32057035
commit 1bd747835f
1 changed files with 41 additions and 11 deletions

View File

@ -852,6 +852,14 @@ class StartServer extends Command
/** /**
* Build the server instance. * Build the server instance.
* *
* Binding retries on EADDRINUSE: during a restart (deploy, supervisor,
* websocket:steer restart-hard) the previous daemon's socket often
* lingers for a few seconds after SIGTERM. Crashing immediately turned
* every restart into a crash-loop of "Failed to listen" errors until the
* port freed up waiting out the handover is the correct behaviour. A
* port that is STILL taken after the retry budget is a real conflict and
* rethrows.
*
* @return void * @return void
*/ */
protected function buildServer() protected function buildServer()
@ -861,22 +869,44 @@ class StartServer extends Command
'port' => $this->option('port'), 'port' => $this->option('port'),
]); ]);
$this->server = new ServerFactory(
$this->option('host'),
$this->option('port')
);
if ($loop = $this->option('loop')) { if ($loop = $this->option('loop')) {
\Log::channel('websocket')->debug('Using injected loop'); \Log::channel('websocket')->debug('Using injected loop');
$this->loop = $loop; $this->loop = $loop;
} }
\Log::channel('websocket')->debug('Configuring server with loop and routes...'); $maxAttempts = 30;
$this->server = $this->server
->setLoop($this->loop) for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
->withRoutes(WebSocketRouter::getRoutes()) try {
->setConsoleOutput($this->output) \Log::channel('websocket')->debug('Configuring server with loop and routes...');
->createServer(); $this->server = (new ServerFactory(
$this->option('host'),
$this->option('port')
))
->setLoop($this->loop)
->withRoutes(WebSocketRouter::getRoutes())
->setConsoleOutput($this->output)
->createServer();
break;
} catch (\RuntimeException $e) {
$isAddrInUse = str_contains($e->getMessage(), 'EADDRINUSE')
|| str_contains($e->getMessage(), 'Address already in use');
if (! $isAddrInUse || $attempt === $maxAttempts) {
throw $e;
}
\Log::channel('websocket')->warning('Port busy (predecessor still shutting down?), retrying bind...', [
'port' => $this->option('port'),
'attempt' => $attempt,
'max_attempts' => $maxAttempts,
]);
$this->components->warn("Port {$this->option('port')} busy, retrying bind ({$attempt}/{$maxAttempts})...");
sleep(1);
}
}
\Log::channel('websocket')->debug('Server created and ready'); \Log::channel('websocket')->debug('Server created and ready');
} }