feat(observability): add Alertmanager email alerts + disk usage PrometheusRules
- Configure Alertmanager SMTP via Mailcow (192.168.1.30:587) - Route critical/warning alerts to alexandre.vazquez@gmail.com - Add DiskUsageHigh (< 20% free), DiskUsageCritical (< 10%), DiskWillFillIn4Hours rules Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -57,6 +57,86 @@ kubelet:
|
||||
sourceLabels: [__name__]
|
||||
regex: storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_bucket
|
||||
|
||||
alertmanager:
|
||||
config:
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
smtp_smarthost: '192.168.1.30:587'
|
||||
smtp_from: 'alerts@alexandre-vazquez.cloud'
|
||||
smtp_auth_username: 'alerts@alexandre-vazquez.cloud'
|
||||
smtp_auth_password: '11BE80k1+jbHBirtVGAlAQ=='
|
||||
smtp_require_tls: true
|
||||
inhibit_rules:
|
||||
- equal: [namespace, alertname]
|
||||
source_matchers: [severity = critical]
|
||||
target_matchers: ['severity =~ warning|info']
|
||||
- equal: [namespace, alertname]
|
||||
source_matchers: [severity = warning]
|
||||
target_matchers: [severity = info]
|
||||
- equal: [namespace]
|
||||
source_matchers: [alertname = InfoInhibitor]
|
||||
target_matchers: [severity = info]
|
||||
receivers:
|
||||
- name: "null"
|
||||
- name: email-alerts
|
||||
email_configs:
|
||||
- to: 'alexandre.vazquez@gmail.com'
|
||||
send_resolved: true
|
||||
headers:
|
||||
Subject: '[HOMELAB] {{ .GroupLabels.alertname }} — {{ .Status | toUpper }}'
|
||||
route:
|
||||
group_by: [namespace, alertname]
|
||||
group_interval: 5m
|
||||
group_wait: 30s
|
||||
receiver: "null"
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
- matchers: ['alertname =~ "InfoInhibitor|Watchdog"']
|
||||
receiver: "null"
|
||||
- matchers: ['severity =~ "critical|warning"']
|
||||
receiver: email-alerts
|
||||
templates:
|
||||
- /etc/alertmanager/config/*.tmpl
|
||||
|
||||
additionalPrometheusRulesMap:
|
||||
disk-usage-alerts:
|
||||
groups:
|
||||
- name: disk.usage
|
||||
rules:
|
||||
- alert: DiskUsageHigh
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
|
||||
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
|
||||
) * 100 < 20
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disco al {{ printf \"%.0f\" $value }}% libre en {{ $labels.instance }}"
|
||||
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 20% libre."
|
||||
- alert: DiskUsageCritical
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
|
||||
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
|
||||
) * 100 < 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disco CRÍTICO en {{ $labels.instance }} ({{ printf \"%.0f\" $value }}% libre)"
|
||||
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 10% libre. ACCIÓN INMEDIATA."
|
||||
- alert: DiskWillFillIn4Hours
|
||||
expr: |
|
||||
predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}[1h], 4*3600) < 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disco se llenará en <4h en {{ $labels.instance }}"
|
||||
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} se llenará en menos de 4 horas."
|
||||
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
evaluationInterval: 60s
|
||||
|
||||
Reference in New Issue
Block a user