From dc64e6540c9ea6aad174b447cb264fb95262e00f Mon Sep 17 00:00:00 2001
From: David Jardin <d.jardin@djumla.de>
Date: Thu, 4 Dec 2025 17:36:50 +0100
Subject: [PATCH 1/2] Improve queue stats (#56)

* increase productio worker count

* Improve queue stats output

* cs fix

* further increase worker count (#55)
---
 app/Health/QueueLengthCheck.php       | 31 +++++++++++++++++++++++++++
 app/Providers/HealthCheckProvider.php |  2 ++
 config/horizon.php                    |  4 ++--
 routes/console.php                    |  1 +
 4 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 app/Health/QueueLengthCheck.php

diff --git a/app/Health/QueueLengthCheck.php b/app/Health/QueueLengthCheck.php
new file mode 100644
index 0000000..4eab7e4
--- /dev/null
+++ b/app/Health/QueueLengthCheck.php
@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Health;
+
+use Laravel\Horizon\Contracts\JobRepository;
+use Spatie\Health\Checks\Check;
+use Spatie\Health\Checks\Result;
+
+class QueueLengthCheck extends Check
+{
+    public function run(): Result
+    {
+        /** @var JobRepository $jobs */
+        $jobs = app(JobRepository::class);
+
+        $result = Result::make()
+            ->appendMeta([
+                'pending' => $jobs->countPending(),
+                'failed' => $jobs->countFailed(),
+                'completed' => $jobs->countCompleted()
+            ]);
+
+        if ($jobs->countPending() > 50000) {
+            return $result
+                ->failed('Job queue larger than 50.000 items')
+                ->shortSummary('Excessive queue length');
+        }
+
+        return $result->ok();
+    }
+}
diff --git a/app/Providers/HealthCheckProvider.php b/app/Providers/HealthCheckProvider.php
index 4123654..954ed16 100644
--- a/app/Providers/HealthCheckProvider.php
+++ b/app/Providers/HealthCheckProvider.php
@@ -2,6 +2,7 @@
 
 namespace App\Providers;
 
+use App\Health\QueueLengthCheck;
 use Illuminate\Support\ServiceProvider;
 use Spatie\CpuLoadHealthCheck\CpuLoadCheck;
 use Spatie\Health\Checks\Checks\DatabaseCheck;
@@ -29,6 +30,7 @@ public function boot(): void
             UsedDiskSpaceCheck::new(),
             DatabaseCheck::new(),
             HorizonCheck::new(),
+            QueueLengthCheck::new(),
             RedisCheck::new(),
             CpuLoadCheck::new()
         ]);
diff --git a/config/horizon.php b/config/horizon.php
index 4f2d7fc..70c4a72 100644
--- a/config/horizon.php
+++ b/config/horizon.php
@@ -211,12 +211,12 @@
     'environments' => [
         'production' => [
             'supervisor-default' => [
-                'maxProcesses' => 15,
+                'maxProcesses' => 20,
                 'balanceMaxShift' => 1,
                 'balanceCooldown' => 3,
             ],
             'supervisor-updates' => [
-                'maxProcesses' => 40,
+                'maxProcesses' => 80,
                 'balanceMaxShift' => 1,
                 'balanceCooldown' => 3,
             ],
diff --git a/routes/console.php b/routes/console.php
index aabe8d1..c18a76a 100644
--- a/routes/console.php
+++ b/routes/console.php
@@ -8,3 +8,4 @@
 Schedule::command(CleanupSitesList::class)->everySixHours();
 Schedule::command(PruneFailedJobsCommand::class)->daily();
 Schedule::command(QueueHealthChecks::class)->everyFifteenMinutes();
+Schedule::command('horizon:snapshot')->everyFiveMinutes();

From e6ea9fc47a9eeb7e49e7c8eb7ad6a13c1ee32f0f Mon Sep 17 00:00:00 2001
From: David Jardin <d.jardin@djumla.de>
Date: Thu, 4 Dec 2025 17:39:07 +0100
Subject: [PATCH 2/2] Improve exception handling and logging (#57)

* increase productio worker count

* improve exceptoin handling and logging

* adjust message

* fix stan error

* include status code changes in exception
---
 app/Jobs/UpdateSite.php | 40 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/app/Jobs/UpdateSite.php b/app/Jobs/UpdateSite.php
index 1ce5bcf..ae9763a 100644
--- a/app/Jobs/UpdateSite.php
+++ b/app/Jobs/UpdateSite.php
@@ -54,7 +54,16 @@ public function handle(): void
         $connection = $this->site->connection;
 
         // Test connection and get current version
-        $healthResult = $connection->checkHealth();
+        try {
+            $healthResult = $connection->checkHealth();
+        } catch (\Throwable $e) {
+            throw new UpdateException(
+                "checkHealth",
+                $e->getMessage(),
+                (int) $e->getCode(),
+                $e instanceof \Exception ? $e : null
+            );
+        }
 
         // Check the version
         if (version_compare($healthResult->cms_version, $this->targetVersion, ">=")) {
@@ -106,7 +115,16 @@ public function handle(): void
             return;
         }
 
-        $prepareResult = $connection->prepareUpdate(["targetVersion" => $this->targetVersion]);
+        try {
+            $prepareResult = $connection->prepareUpdate(["targetVersion" => $this->targetVersion]);
+        } catch (\Throwable $e) {
+            throw new UpdateException(
+                "prepareUpdate",
+                $e->getMessage(),
+                (int) $e->getCode(),
+                $e instanceof \Exception ? $e : null
+            );
+        }
 
         // Perform the actual extraction
         try {
@@ -139,7 +157,12 @@ public function handle(): void
         if ($afterUpdateCode !== $this->preUpdateCode) {
             throw new UpdateException(
                 "afterUpdate",
-                "Status code has changed after update for site: " . $this->site->id
+                sprintf(
+                    "Status code has changed from %s to %s after update for site: %s",
+                    $this->preUpdateCode,
+                    $afterUpdateCode,
+                    $this->site->id
+                )
             );
         }
 
@@ -213,10 +236,19 @@ public function failed(\Exception $exception): void
         $connection = $this->site->connection;
 
         // Get base execption information
-        $failedStep = $exception instanceof UpdateException ? $exception->getStep() : null;
+        $failedStep = $exception instanceof UpdateException ? $exception->getStep() : get_class($exception);
         $failedMessage = $exception->getMessage();
         $failedTrace = $exception->getTraceAsString();
 
+        // Log response body
+        if ($exception instanceof RequestException) {
+            $failedMessage .= "\n Response Body: " . $exception->getResponse()?->getBody();
+        }
+
+        if ($exception->getPrevious() instanceof RequestException) {
+            $failedMessage .= "\n Response Body: " . $exception->getPrevious()->getResponse()?->getBody();
+        }
+
         // Notify users
         try {
             $connection->notificationFailed(["fromVersion" => $this->site->cms_version, "toVersion" => $this->targetVersion]);