ha-prometheus/roles/ha-monitoring/templates/configs/prometheus/alert_rules.yml

59 lines
2.0 KiB
YAML

groups:
- name: Prometheus self-monitoring
rules:
- alert: Prometheus target missing
expr: 'up == 0'
annotations:
description: A Prometheus target has disappeared. An exporter might be crashed.
labels:
severity: critical
for: 2m
- alert: Prometheus AlertManager config not synced
expr: 'count(count_values("config_hash", alertmanager_config_hash)) > 1'
annotations:
description: Configurations of AlertManager cluster instances are out of sync
labels:
severity: warning
- alert: Prometheus target scraping slow
expr: 'prometheus_target_interval_length_seconds{quantile="0.9"} > 60'
annotations:
description: Prometheus is scraping exporters slowly
labels:
severity: warning
for: 5m
- name: Host and hardware
rules:
- alert: Host out of memory
expr: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10'
annotations:
description: Node memory is filling up (< 10% left)
labels:
severity: warning
for: 2m
- alert: Host out of disk space
expr: '(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0'
annotations:
description: Disk is almost full (< 10% left)
labels:
severity: warning
for: 2m
- alert: Host high CPU load
expr: '100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80'
annotations:
description: CPU load is > 80%
labels:
severity: warning
for: 2m
- alert: Host OOM kill detected
expr: 'increase(node_vmstat_oom_kill[1m]) > 0'
annotations:
description: OOM kill detected
labels:
severity: warning
- alert: Host clock skew
expr: '(node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)'
annotations:
description: 'Clock skew detected. Clock is out of sync.'
labels:
severity: warning
for: 2m