Skip to content

Commit 75accd6

Browse files
committed
Improve implementation of grace period
1 parent f270eac commit 75accd6

File tree

3 files changed

+28
-18
lines changed

3 files changed

+28
-18
lines changed

src/clickhouse.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import type { Group, HistoryRecord, IntervalConfig, Monitor, PulseRecord, Status
66
import { missingPulseDetector } from "./missing-pulse-detector";
77
import { NotificationManager } from "./notifications";
88
import { cache } from "./cache";
9-
import { formatDateTimeISOCompact, formatDateTimeISOString } from "./times";
9+
import { formatDateTimeISOCompact, formatDateTimeISOString, GRACE_PERIOD, isInGracePeriod, STARTUP_TIME } from "./times";
1010

1111
export const eventEmitter = new EventEmitter();
1212

@@ -451,7 +451,17 @@ export async function updateGroupStatus(groupId: string): Promise<void> {
451451

452452
cache.setStatus(groupId, groupStatus);
453453

454-
if (previousStatus && previousStatus !== status && !isStartup && group.notificationChannels && group.notificationChannels.length > 0) {
454+
// Log if we're skipping notifications during grace period
455+
if (!previousStatus && isInGracePeriod()) {
456+
Logger.info("Skipping group notifications during grace period", {
457+
groupId,
458+
groupName: group.name,
459+
status,
460+
gracePeriodRemaining: Math.round((GRACE_PERIOD - (Date.now() - STARTUP_TIME)) / 1000) + "s",
461+
});
462+
}
463+
464+
if (previousStatus && previousStatus !== status && !isInGracePeriod() && group.notificationChannels && group.notificationChannels.length > 0) {
455465
const notificationManager = new NotificationManager(config.notifications || { channels: {} });
456466

457467
if (status === "down" || status === "degraded") {

src/missing-pulse-detector.ts

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { storePulse, eventEmitter } from "./clickhouse";
33
import { config } from "./config";
44
import { Logger } from "./logger";
55
import { NotificationManager } from "./notifications";
6+
import { GRACE_PERIOD, isInGracePeriod, STARTUP_TIME } from "./times";
67
import type { MissingPulseDetectorOptions, Monitor, NotificationsConfig } from "./types";
78

89
export class MissingPulseDetector {
@@ -13,23 +14,12 @@ export class MissingPulseDetector {
1314
private consecutiveDownCounts: Map<string, number> = new Map();
1415
private lastNotificationDownCount: Map<string, number> = new Map();
1516
private notificationManager: NotificationManager;
16-
private startupTime: number;
17-
private gracePeriod: number;
1817

1918
constructor(options: MissingPulseDetectorOptions = {}) {
2019
this.checkInterval = options.checkInterval || 30000; // Check every 30 seconds globally
21-
this.gracePeriod = 300000; // 5 minutes default
22-
this.startupTime = Date.now();
2320
this.notificationManager = new NotificationManager(config.notifications || { channels: {} });
2421
}
2522

26-
/**
27-
* Check if we're still in the startup grace period
28-
*/
29-
private isInGracePeriod(): boolean {
30-
return Date.now() - this.startupTime < this.gracePeriod;
31-
}
32-
3323
/**
3424
* Start the missing pulse detection
3525
*/
@@ -88,17 +78,17 @@ export class MissingPulseDetector {
8878
// No status data yet - monitor hasn't sent its first pulse
8979
if (!status) {
9080
// Check if enough time has passed since startup to consider this a problem
91-
const timeSinceStartup = now - this.startupTime;
81+
const timeSinceStartup = now - STARTUP_TIME;
9282
const expectedInterval = monitor.interval * 1000;
9383
const maxAllowedInterval = expectedInterval * monitor.toleranceFactor;
9484

9585
// Only start checking after grace period + one full interval
96-
if (timeSinceStartup > this.gracePeriod + maxAllowedInterval) {
86+
if (timeSinceStartup > GRACE_PERIOD + maxAllowedInterval) {
9787
Logger.warn("Monitor has never sent a pulse", {
9888
monitorId: monitor.id,
9989
monitorName: monitor.name,
10090
timeSinceStartup: Math.round(timeSinceStartup / 1000) + "s",
101-
gracePeriod: this.gracePeriod / 1000 + "s",
91+
gracePeriod: GRACE_PERIOD / 1000 + "s",
10292
});
10393

10494
await this.handleMissingPulse(monitor, timeSinceStartup, expectedInterval);
@@ -150,11 +140,11 @@ export class MissingPulseDetector {
150140
missedIntervals,
151141
consecutiveMisses: missedCount,
152142
maxRetries: monitor.maxRetries,
153-
inGracePeriod: this.isInGracePeriod(),
143+
inGracePeriod: isInGracePeriod(),
154144
});
155145

156146
// Don't mark monitors as down during grace period
157-
if (this.isInGracePeriod()) {
147+
if (isInGracePeriod()) {
158148
Logger.info("Skipping status change during grace period", {
159149
monitorId: monitor.id,
160150
monitorName: monitor.name,

src/times.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
export const STARTUP_TIME = Date.now();
2+
export const GRACE_PERIOD = 300000; // 5 minutes
3+
4+
/**
5+
* Check if we're still in the startup grace period
6+
*/
7+
export function isInGracePeriod(): boolean {
8+
return Date.now() - STARTUP_TIME < GRACE_PERIOD;
9+
}
10+
111
/**
212
Output Example: `2001-10-15 20:56` or `2001-10-15 20:56:27`
313
*/

0 commit comments

Comments
 (0)