Skip to content

Instantly share code, notes, and snippets.

@hagen1778
Created August 8, 2025 11:59
Show Gist options
  • Select an option

  • Save hagen1778/e5e1ac57d345c01ebd9b0041cdffafb3 to your computer and use it in GitHub Desktop.

Select an option

Save hagen1778/e5e1ac57d345c01ebd9b0041cdffafb3 to your computer and use it in GitHub Desktop.
diff --git a/app/vmalert/rule/alerting.go b/app/vmalert/rule/alerting.go
index 07ccce0e4c..2a9695bbfc 100644
--- a/app/vmalert/rule/alerting.go
+++ b/app/vmalert/rule/alerting.go
@@ -425,17 +425,36 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
// template labels and annotations before updating ar.alerts,
// since they could use `query` function which takes a while to execute,
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6079.
- expandedLabels := make([]*labelSet, len(res.Data))
- expandedAnnotations := make([]map[string]string, len(res.Data))
- for i, m := range res.Data {
- ls, as, err := ar.expandTemplates(m, qFn, ts)
+ type alertMetadata struct {
+ value float64
+ labels *labelSet
+ annotations map[string]string
+ }
+ ams := make(map[uint64]*alertMetadata, len(res.Data))
+ ar.alertsMu.RLock()
+ for _, m := range res.Data {
+ ls, err := ar.toLabels(m, qFn)
+ if err != nil {
+ curState.Err = fmt.Errorf("failed to expand labels: %w", err)
+ return nil, curState.Err
+ }
+ alertID := hash(ls.processed)
+ activeAt := ts
+ if a, ok := ar.alerts[alertID]; ok {
+ activeAt = a.ActiveAt
+ }
+ as, err := ar.expandTemplates(m, ls, qFn, activeAt)
if err != nil {
- curState.Err = fmt.Errorf("failed to expand templates: %w", err)
+ curState.Err = fmt.Errorf("failed to expand annotations: %w", err)
return nil, curState.Err
}
- expandedLabels[i] = ls
- expandedAnnotations[i] = as
+ ams[alertID] = &alertMetadata{
+ value: m.Values[0],
+ labels: ls,
+ annotations: as,
+ }
}
+ ar.alertsMu.RUnlock()
ar.alertsMu.Lock()
defer ar.alertsMu.Unlock()
@@ -450,9 +469,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
updated := make(map[uint64]struct{})
// update list of active alerts
- for i, m := range res.Data {
- labels, annotations := expandedLabels[i], expandedAnnotations[i]
- alertID := hash(labels.processed)
+ for alertID, metadata := range ams {
+ labels, annotations := metadata.labels, metadata.annotations
if _, ok := updated[alertID]; ok {
// duplicate may be caused the removal of `__name__` label
curState.Err = fmt.Errorf("labels %v: %w", labels.processed, errDuplicate)
@@ -468,12 +486,12 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
a.ActiveAt = ts
ar.logDebugf(ts, a, "INACTIVE => PENDING")
}
- a.Value = m.Values[0]
+ a.Value = metadata.value
a.Annotations = annotations
a.KeepFiringSince = time.Time{}
continue
}
- a := ar.newAlert(m, ts, labels.processed, annotations)
+ a := ar.newAlert(metadata.value, ts, labels.processed, annotations)
a.ID = alertID
a.State = notifier.StatePending
ar.alerts[alertID] = a
@@ -536,12 +554,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
return append(tss, ar.toTimeSeries(ts.Unix())...), nil
}
-func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
- ls, err := ar.toLabels(m, qFn)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
- }
-
+func (ar *AlertingRule) expandTemplates(m datasource.Metric, ls *labelSet, qFn templates.QueryFn, activeAt time.Time) (map[string]string, error) {
tplData := notifier.AlertTplData{
Value: m.Values[0],
Type: ar.Type.String(),
@@ -549,14 +562,14 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
Expr: ar.Expr,
AlertID: hash(ls.processed),
GroupID: ar.GroupID,
- ActiveAt: ts,
+ ActiveAt: activeAt,
For: ar.For,
}
as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
if err != nil {
- return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
+ return nil, fmt.Errorf("failed to template annotations: %w", err)
}
- return ls, as, nil
+ return as, nil
}
// toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts
@@ -593,7 +606,7 @@ func hash(labels map[string]string) uint64 {
return hash.Sum64()
}
-func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, annotations map[string]string) *notifier.Alert {
+func (ar *AlertingRule) newAlert(v float64, start time.Time, labels, annotations map[string]string) *notifier.Alert {
as := make(map[string]string)
if annotations != nil {
as = annotations
@@ -609,7 +622,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
Expr: ar.Expr,
For: ar.For,
ActiveAt: start,
- Value: m.Values[0],
+ Value: v,
Labels: ls,
Annotations: as,
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment