From 292c4cc069f97c9539cab5518dcf93431646d5a8 Mon Sep 17 00:00:00 2001 From: zjncs <18910855655@163.com> Date: Fri, 4 Sep 2026 11:51:33 +0800 Subject: [PATCH] fix(alerter): fall back to default repeat interval when group rule was deleted When a converge rule is deleted, renamed or disabled while its group cache still holds firing alerts, sendGroupAlert looked up the rule and dereferenced it without a null check. The NPE aborted the groupCacheMap iteration in runCheckAndSendGroups every check cycle, so every group ordered after the orphaned one stopped receiving converged alerts until the process restarted. Fall back to DEFAULT_REPEAT_INTERVAL, mirroring the null handling that shouldSendGroup already applies for the group wait and group interval. Signed-off-by: zjncs <18910855655@163.com> --- .../alert/reduce/AlarmGroupReduce.java | 4 ++- .../alert/reduce/AlarmGroupReduceTest.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index fac5f7b268f..79c9b73493e 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -291,7 +291,9 @@ private void sendGroupAlert(GroupAlertCache cache) { // For firing alerts, check repeat interval if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) { AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName()); - long repeatInterval = ruleConfig.getRepeatInterval() != null + // The rule may have been deleted, renamed or disabled while this group cache + // still holds firing alerts; fall back to the default interval like shouldSendGroup. + long repeatInterval = ruleConfig != null && ruleConfig.getRepeatInterval() != null ? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL; // Skip if within repeat interval. The throttle only suppresses repeated firing diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index 4611d521383..519a3221e49 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.lang.reflect.Field; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; @@ -212,6 +213,37 @@ void whenMemberRecoversInsideRepeatInterval_recoveryMustStillBeEmitted() { "memory recovery was silently dropped by the firing repeat-interval throttle"); } + /** + * Regression: deleting (or renaming/disabling) a converge rule while its group cache still + * holds a firing alert must not break the periodic group dispatch with an NPE. The orphaned + * group has to fall back to the default repeat interval, like shouldSendGroup already does + * for the group wait/interval. + */ + @Test + void whenRuleDeleted_firingGroupCacheMustStillBeDispatched() throws Exception { + alarmGroupReduce.refreshGroupDefines(Collections.singletonList(groupRule())); + + alarmGroupReduce.processGroupAlert(alert("cpu", "firing", "host1")); + + // The rule is deleted or disabled; refresh loads only enabled rules, orphaning the cache. + alarmGroupReduce.refreshGroupDefines(Collections.emptyList()); + + // Age the cache past the default group wait so the periodic check picks the group up. + Field cachesField = AlarmGroupReduce.class.getDeclaredField("groupCacheMap"); + cachesField.setAccessible(true); + Map caches = (Map) cachesField.get(alarmGroupReduce); + for (Object cache : caches.values()) { + Field createTimeField = cache.getClass().getDeclaredField("createTime"); + createTimeField.setAccessible(true); + createTimeField.setLong(cache, System.currentTimeMillis() - 60_000); + } + + alarmGroupReduce.runCheckAndSendGroups(); + + verify(alarmInhibitReduce, atLeastOnce()).inhibitAlarm(argThat(group -> + group.getAlerts().stream().anyMatch(a -> "cpu".equals(a.getFingerprint())))); + } + private AlertGroupConverge groupRule() { AlertGroupConverge rule = new AlertGroupConverge(); rule.setName("test-rule");