feat(observability): add Alertmanager email alerts + disk usage PrometheusRules
- Configure Alertmanager SMTP via Mailcow (192.168.1.30:587) - Route critical/warning alerts to alexandre.vazquez@gmail.com - Add DiskUsageHigh (< 20% free), DiskUsageCritical (< 10%), DiskWillFillIn4Hours rules Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -57,6 +57,86 @@ kubelet:
|
|||||||
sourceLabels: [__name__]
|
sourceLabels: [__name__]
|
||||||
regex: storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_bucket
|
regex: storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_bucket
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
config:
|
||||||
|
global:
|
||||||
|
resolve_timeout: 5m
|
||||||
|
smtp_smarthost: '192.168.1.30:587'
|
||||||
|
smtp_from: 'alerts@alexandre-vazquez.cloud'
|
||||||
|
smtp_auth_username: 'alerts@alexandre-vazquez.cloud'
|
||||||
|
smtp_auth_password: '11BE80k1+jbHBirtVGAlAQ=='
|
||||||
|
smtp_require_tls: true
|
||||||
|
inhibit_rules:
|
||||||
|
- equal: [namespace, alertname]
|
||||||
|
source_matchers: [severity = critical]
|
||||||
|
target_matchers: ['severity =~ warning|info']
|
||||||
|
- equal: [namespace, alertname]
|
||||||
|
source_matchers: [severity = warning]
|
||||||
|
target_matchers: [severity = info]
|
||||||
|
- equal: [namespace]
|
||||||
|
source_matchers: [alertname = InfoInhibitor]
|
||||||
|
target_matchers: [severity = info]
|
||||||
|
receivers:
|
||||||
|
- name: "null"
|
||||||
|
- name: email-alerts
|
||||||
|
email_configs:
|
||||||
|
- to: 'alexandre.vazquez@gmail.com'
|
||||||
|
send_resolved: true
|
||||||
|
headers:
|
||||||
|
Subject: '[HOMELAB] {{ .GroupLabels.alertname }} — {{ .Status | toUpper }}'
|
||||||
|
route:
|
||||||
|
group_by: [namespace, alertname]
|
||||||
|
group_interval: 5m
|
||||||
|
group_wait: 30s
|
||||||
|
receiver: "null"
|
||||||
|
repeat_interval: 4h
|
||||||
|
routes:
|
||||||
|
- matchers: ['alertname =~ "InfoInhibitor|Watchdog"']
|
||||||
|
receiver: "null"
|
||||||
|
- matchers: ['severity =~ "critical|warning"']
|
||||||
|
receiver: email-alerts
|
||||||
|
templates:
|
||||||
|
- /etc/alertmanager/config/*.tmpl
|
||||||
|
|
||||||
|
additionalPrometheusRulesMap:
|
||||||
|
disk-usage-alerts:
|
||||||
|
groups:
|
||||||
|
- name: disk.usage
|
||||||
|
rules:
|
||||||
|
- alert: DiskUsageHigh
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
|
||||||
|
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
|
||||||
|
) * 100 < 20
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Disco al {{ printf \"%.0f\" $value }}% libre en {{ $labels.instance }}"
|
||||||
|
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 20% libre."
|
||||||
|
- alert: DiskUsageCritical
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
|
||||||
|
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
|
||||||
|
) * 100 < 10
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Disco CRÍTICO en {{ $labels.instance }} ({{ printf \"%.0f\" $value }}% libre)"
|
||||||
|
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 10% libre. ACCIÓN INMEDIATA."
|
||||||
|
- alert: DiskWillFillIn4Hours
|
||||||
|
expr: |
|
||||||
|
predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}[1h], 4*3600) < 0
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Disco se llenará en <4h en {{ $labels.instance }}"
|
||||||
|
description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} se llenará en menos de 4 horas."
|
||||||
|
|
||||||
prometheus:
|
prometheus:
|
||||||
prometheusSpec:
|
prometheusSpec:
|
||||||
evaluationInterval: 60s
|
evaluationInterval: 60s
|
||||||
|
|||||||
Reference in New Issue
Block a user