Develop

app/Health/QueueLengthCheck.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Health;
+
+use Laravel\Horizon\Contracts\JobRepository;
+use Spatie\Health\Checks\Check;
+use Spatie\Health\Checks\Result;
+
+class QueueLengthCheck extends Check
+{
+    public function run(): Result
+    {
+        /** @var JobRepository $jobs */
+        $jobs = app(JobRepository::class);
+
+        $result = Result::make()
+            ->appendMeta([
+                'pending' => $jobs->countPending(),
+                'failed' => $jobs->countFailed(),
+                'completed' => $jobs->countCompleted()
+            ]);
+
+        if ($jobs->countPending() > 50000) {
+            return $result
+                ->failed('Job queue larger than 50.000 items')
+                ->shortSummary('Excessive queue length');
+        }
+
+        return $result->ok();
+    }
+}

app/Jobs/UpdateSite.php

@@ -54,7 +54,16 @@ public function handle(): void
         $connection = $this->site->connection;
 
         // Test connection and get current version
-        $healthResult = $connection->checkHealth();
+        try {
+            $healthResult = $connection->checkHealth();
+        } catch (\Throwable $e) {
+            throw new UpdateException(
+                "checkHealth",
+                $e->getMessage(),
+                (int) $e->getCode(),
+                $e instanceof \Exception ? $e : null
+            );
+        }
 
         // Check the version
         if (version_compare($healthResult->cms_version, $this->targetVersion, ">=")) {
@@ -106,7 +115,16 @@ public function handle(): void
             return;
         }
 
-        $prepareResult = $connection->prepareUpdate(["targetVersion" => $this->targetVersion]);
+        try {
+            $prepareResult = $connection->prepareUpdate(["targetVersion" => $this->targetVersion]);
+        } catch (\Throwable $e) {
+            throw new UpdateException(
+                "prepareUpdate",
+                $e->getMessage(),
+                (int) $e->getCode(),
+                $e instanceof \Exception ? $e : null
+            );
+        }
 
         // Perform the actual extraction
         try {
@@ -139,7 +157,12 @@ public function handle(): void
         if ($afterUpdateCode !== $this->preUpdateCode) {
             throw new UpdateException(
                 "afterUpdate",
-                "Status code has changed after update for site: " . $this->site->id
+                sprintf(
+                    "Status code has changed from %s to %s after update for site: %s",
+                    $this->preUpdateCode,
+                    $afterUpdateCode,
+                    $this->site->id
+                )
             );
         }
 
@@ -213,10 +236,19 @@ public function failed(\Exception $exception): void
         $connection = $this->site->connection;
 
         // Get base execption information
-        $failedStep = $exception instanceof UpdateException ? $exception->getStep() : null;
+        $failedStep = $exception instanceof UpdateException ? $exception->getStep() : get_class($exception);
         $failedMessage = $exception->getMessage();
         $failedTrace = $exception->getTraceAsString();
 
+        // Log response body
+        if ($exception instanceof RequestException) {
+            $failedMessage .= "\n Response Body: " . $exception->getResponse()?->getBody();
+        }
+
+        if ($exception->getPrevious() instanceof RequestException) {
+            $failedMessage .= "\n Response Body: " . $exception->getPrevious()->getResponse()?->getBody();
+        }
+
         // Notify users
         try {
             $connection->notificationFailed(["fromVersion" => $this->site->cms_version, "toVersion" => $this->targetVersion]);

app/Providers/HealthCheckProvider.php

@@ -2,6 +2,7 @@
 
 namespace App\Providers;
 
+use App\Health\QueueLengthCheck;
 use Illuminate\Support\ServiceProvider;
 use Spatie\CpuLoadHealthCheck\CpuLoadCheck;
 use Spatie\Health\Checks\Checks\DatabaseCheck;
@@ -29,6 +30,7 @@ public function boot(): void
             UsedDiskSpaceCheck::new(),
             DatabaseCheck::new(),
             HorizonCheck::new(),
+            QueueLengthCheck::new(),
             RedisCheck::new(),
             CpuLoadCheck::new()
         ]);

config/horizon.php

@@ -211,12 +211,12 @@
     'environments' => [
         'production' => [
             'supervisor-default' => [
-                'maxProcesses' => 15,
+                'maxProcesses' => 20,
                 'balanceMaxShift' => 1,
                 'balanceCooldown' => 3,
             ],
             'supervisor-updates' => [
-                'maxProcesses' => 40,
+                'maxProcesses' => 80,
                 'balanceMaxShift' => 1,
                 'balanceCooldown' => 3,
             ],

routes/console.php

@@ -8,3 +8,4 @@
 Schedule::command(CleanupSitesList::class)->everySixHours();
 Schedule::command(PruneFailedJobsCommand::class)->daily();
 Schedule::command(QueueHealthChecks::class)->everyFifteenMinutes();
+Schedule::command('horizon:snapshot')->everyFiveMinutes();