diff --git a/app/Health/QueueLengthCheck.php b/app/Health/QueueLengthCheck.php new file mode 100644 index 0000000..4eab7e4 --- /dev/null +++ b/app/Health/QueueLengthCheck.php @@ -0,0 +1,31 @@ +appendMeta([ + 'pending' => $jobs->countPending(), + 'failed' => $jobs->countFailed(), + 'completed' => $jobs->countCompleted() + ]); + + if ($jobs->countPending() > 50000) { + return $result + ->failed('Job queue larger than 50.000 items') + ->shortSummary('Excessive queue length'); + } + + return $result->ok(); + } +} diff --git a/app/Providers/HealthCheckProvider.php b/app/Providers/HealthCheckProvider.php index 4123654..954ed16 100644 --- a/app/Providers/HealthCheckProvider.php +++ b/app/Providers/HealthCheckProvider.php @@ -2,6 +2,7 @@ namespace App\Providers; +use App\Health\QueueLengthCheck; use Illuminate\Support\ServiceProvider; use Spatie\CpuLoadHealthCheck\CpuLoadCheck; use Spatie\Health\Checks\Checks\DatabaseCheck; @@ -29,6 +30,7 @@ public function boot(): void UsedDiskSpaceCheck::new(), DatabaseCheck::new(), HorizonCheck::new(), + QueueLengthCheck::new(), RedisCheck::new(), CpuLoadCheck::new() ]); diff --git a/config/horizon.php b/config/horizon.php index 4f2d7fc..70c4a72 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -211,12 +211,12 @@ 'environments' => [ 'production' => [ 'supervisor-default' => [ - 'maxProcesses' => 15, + 'maxProcesses' => 20, 'balanceMaxShift' => 1, 'balanceCooldown' => 3, ], 'supervisor-updates' => [ - 'maxProcesses' => 40, + 'maxProcesses' => 80, 'balanceMaxShift' => 1, 'balanceCooldown' => 3, ], diff --git a/routes/console.php b/routes/console.php index aabe8d1..c18a76a 100644 --- a/routes/console.php +++ b/routes/console.php @@ -8,3 +8,4 @@ Schedule::command(CleanupSitesList::class)->everySixHours(); Schedule::command(PruneFailedJobsCommand::class)->daily(); Schedule::command(QueueHealthChecks::class)->everyFifteenMinutes(); +Schedule::command('horizon:snapshot')->everyFiveMinutes();