From e535e4fc2df4e5ad6d8f59a2c51ef24c038fe9a0 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Fri, 25 Sep 2026 09:49:05 +0200 Subject: [PATCH 1/2] Group ingestion source error alerts by operation A global error ratio hides a stuck getRaftId behind getRaftLog volume. Summing by op pages that setup failure and a later getRaftLog failure on the same 3% / 5% alerts. Issue: BB-605 --- monitoring/ingestion/ingestion-producer-alert.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/monitoring/ingestion/ingestion-producer-alert.yaml b/monitoring/ingestion/ingestion-producer-alert.yaml index 73ec1073b4..6d32a24f5b 100644 --- a/monitoring/ingestion/ingestion-producer-alert.yaml +++ b/monitoring/ingestion/ingestion-producer-alert.yaml @@ -70,11 +70,13 @@ groups: description: "Ingestion producer has not queued any messages in the last 5 minutes" summary: "Ingestion producer has not queued messages" + # Ratio per source op so a stuck getRaftId is not hidden by getRaftLog + # volume, and a later getRaftLog failure is the same alert on that op. - alert: IngestionProducerSourceErrorRate3Percent Expr: | - sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) / - sum(rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) > 0.03 For: "5m" Labels: @@ -86,9 +88,9 @@ groups: - alert: IngestionProducerSourceErrorRate3Percent Expr: | - sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) / - sum(rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + sum by (op) (rate(s3_ingestion_source_operations_total{namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) > 0.05 For: "5m" Labels: From 4cb914f9332075eefa835d168796533b2510ef62 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Fri, 25 Sep 2026 09:50:13 +0200 Subject: [PATCH 2/2] Test the ingestion producer source error alerts The ingestion alerts had no render-and-test step. Cover a stuck getRaftId, a recovering setup, and a getRaftLog failure on the same 3% / 5% rules. Issue: BB-605 --- .github/workflows/alerts.yaml | 11 +++ .../ingestion-producer-alert.test.yaml | 84 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 monitoring/ingestion/ingestion-producer-alert.test.yaml diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 4ebbb7445c..3e3ba043bc 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -74,6 +74,17 @@ jobs: notification_processor_replicas=2 github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test ingestion producer + uses: scality/action-prom-render-test@1.0.3 + with: + alert_file_path: monitoring/ingestion/ingestion-producer-alert.yaml + test_file_path: monitoring/ingestion/ingestion-producer-alert.test.yaml + alert_inputs: | + namespace=zenko + job_ingestion_producer=artesca-data-backbeat-ingestion-producer-headless + replicas=1 + github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test oplog populator uses: scality/action-prom-render-test@1.0.3 with: diff --git a/monitoring/ingestion/ingestion-producer-alert.test.yaml b/monitoring/ingestion/ingestion-producer-alert.test.yaml new file mode 100644 index 0000000000..c400e3126e --- /dev/null +++ b/monitoring/ingestion/ingestion-producer-alert.test.yaml @@ -0,0 +1,84 @@ +evaluation_interval: 1m +rule_files: + - ingestion-producer-alert.rendered.yaml + +tests: + + # getRaftId errors stay at 100% on that op even when getRaftLog volume is high. + # The warning and critical rules share one name, so Alertmanager raises the + # severity instead of sending two alerts. + - name: Ingestion Producer Source Setup Failing + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0+12x20 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="success"} + values: 0+12000x20 + alert_rule_test: + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 5m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 12m + exp_alerts: + - exp_labels: + severity: warning + op: getRaftId + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" + - exp_labels: + severity: critical + op: getRaftId + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" + + # A source that fails to set up while its location restarts, then recovers, + # must not alert. + - name: Ingestion Producer Source Setup Recovering + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0 12 24 24x17 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"} + values: 0 0 0 1+1x17 + alert_rule_test: + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 8m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 15m + exp_alerts: [] + + # A later getRaftLog failure is the same 3% / 5% alert on that op. + - name: Ingestion Producer Source Read Failing + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"} + values: 0+12x20 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="error"} + values: 0+12x20 + alert_rule_test: + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 5m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 12m + exp_alerts: + - exp_labels: + severity: warning + op: getRaftLog + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased" + - exp_labels: + severity: critical + op: getRaftLog + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes" + summary: "Ingestion producer source error rate has increased"