feat(observability): add Alertmanager email alerts + disk usage PrometheusRules

- Configure Alertmanager SMTP via Mailcow (192.168.1.30:587)
- Route critical/warning alerts to alexandre.vazquez@gmail.com
- Add DiskUsageHigh (< 20% free), DiskUsageCritical (< 10%), DiskWillFillIn4Hours rules

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
alexandrev-tibco
2026-05-11 11:53:30 +02:00
parent be51b842ff
commit 88fcce4962
@@ -57,6 +57,86 @@ kubelet:
sourceLabels: [__name__]
regex: storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_bucket
alertmanager:
config:
global:
resolve_timeout: 5m
smtp_smarthost: '192.168.1.30:587'
smtp_from: 'alerts@alexandre-vazquez.cloud'
smtp_auth_username: 'alerts@alexandre-vazquez.cloud'
smtp_auth_password: '11BE80k1+jbHBirtVGAlAQ=='
smtp_require_tls: true
inhibit_rules:
- equal: [namespace, alertname]
source_matchers: [severity = critical]
target_matchers: ['severity =~ warning|info']
- equal: [namespace, alertname]
source_matchers: [severity = warning]
target_matchers: [severity = info]
- equal: [namespace]
source_matchers: [alertname = InfoInhibitor]
target_matchers: [severity = info]
receivers:
- name: "null"
- name: email-alerts
email_configs:
- to: 'alexandre.vazquez@gmail.com'
send_resolved: true
headers:
Subject: '[HOMELAB] {{ .GroupLabels.alertname }} — {{ .Status | toUpper }}'
route:
group_by: [namespace, alertname]
group_interval: 5m
group_wait: 30s
receiver: "null"
repeat_interval: 4h
routes:
- matchers: ['alertname =~ "InfoInhibitor|Watchdog"']
receiver: "null"
- matchers: ['severity =~ "critical|warning"']
receiver: email-alerts
templates:
- /etc/alertmanager/config/*.tmpl
additionalPrometheusRulesMap:
disk-usage-alerts:
groups:
- name: disk.usage
rules:
- alert: DiskUsageHigh
expr: |
(
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
) * 100 < 20
for: 5m
labels:
severity: warning
annotations:
summary: "Disco al {{ printf \"%.0f\" $value }}% libre en {{ $labels.instance }}"
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 20% libre."
- alert: DiskUsageCritical
expr: |
(
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
) * 100 < 10
for: 2m
labels:
severity: critical
annotations:
summary: "Disco CRÍTICO en {{ $labels.instance }} ({{ printf \"%.0f\" $value }}% libre)"
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 10% libre. ACCIÓN INMEDIATA."
- alert: DiskWillFillIn4Hours
expr: |
predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}[1h], 4*3600) < 0
for: 10m
labels:
severity: warning
annotations:
summary: "Disco se llenará en <4h en {{ $labels.instance }}"
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} se llenará en menos de 4 horas."
prometheus:
prometheusSpec:
evaluationInterval: 60s