From 88fcce496296e8954c03c687f8802aa50adc9fc4 Mon Sep 17 00:00:00 2001
From: alexandrev-tibco <alexandre.vazquez@cloud.com>
Date: Mon, 11 May 2026 11:53:30 +0200
Subject: [PATCH] feat(observability): add Alertmanager email alerts + disk
 usage PrometheusRules

- Configure Alertmanager SMTP via Mailcow (192.168.1.30:587)
- Route critical/warning alerts to alexandre.vazquez@gmail.com
- Add DiskUsageHigh (< 20% free), DiskUsageCritical (< 10%), DiskWillFillIn4Hours rules

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../observability/kube-prom-stack-values.yaml | 80 +++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/apps/observability/kube-prom-stack-values.yaml b/apps/observability/kube-prom-stack-values.yaml
index d66b204..ee353ee 100644
--- a/apps/observability/kube-prom-stack-values.yaml
+++ b/apps/observability/kube-prom-stack-values.yaml
@@ -57,6 +57,86 @@ kubelet:
       sourceLabels: [__name__]
       regex: storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_bucket
 
+alertmanager:
+  config:
+    global:
+      resolve_timeout: 5m
+      smtp_smarthost: '192.168.1.30:587'
+      smtp_from: 'alerts@alexandre-vazquez.cloud'
+      smtp_auth_username: 'alerts@alexandre-vazquez.cloud'
+      smtp_auth_password: '11BE80k1+jbHBirtVGAlAQ=='
+      smtp_require_tls: true
+    inhibit_rules:
+    - equal: [namespace, alertname]
+      source_matchers: [severity = critical]
+      target_matchers: ['severity =~ warning|info']
+    - equal: [namespace, alertname]
+      source_matchers: [severity = warning]
+      target_matchers: [severity = info]
+    - equal: [namespace]
+      source_matchers: [alertname = InfoInhibitor]
+      target_matchers: [severity = info]
+    receivers:
+    - name: "null"
+    - name: email-alerts
+      email_configs:
+      - to: 'alexandre.vazquez@gmail.com'
+        send_resolved: true
+        headers:
+          Subject: '[HOMELAB] {{ .GroupLabels.alertname }} — {{ .Status | toUpper }}'
+    route:
+      group_by: [namespace, alertname]
+      group_interval: 5m
+      group_wait: 30s
+      receiver: "null"
+      repeat_interval: 4h
+      routes:
+      - matchers: ['alertname =~ "InfoInhibitor|Watchdog"']
+        receiver: "null"
+      - matchers: ['severity =~ "critical|warning"']
+        receiver: email-alerts
+    templates:
+    - /etc/alertmanager/config/*.tmpl
+
+additionalPrometheusRulesMap:
+  disk-usage-alerts:
+    groups:
+    - name: disk.usage
+      rules:
+      - alert: DiskUsageHigh
+        expr: |
+          (
+            node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
+            node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
+          ) * 100 < 20
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Disco al {{ printf \"%.0f\" $value }}% libre en {{ $labels.instance }}"
+          description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 20% libre."
+      - alert: DiskUsageCritical
+        expr: |
+          (
+            node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
+            node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
+          ) * 100 < 10
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disco CRÍTICO en {{ $labels.instance }} ({{ printf \"%.0f\" $value }}% libre)"
+          description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 10% libre. ACCIÓN INMEDIATA."
+      - alert: DiskWillFillIn4Hours
+        expr: |
+          predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}[1h], 4*3600) < 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Disco se llenará en <4h en {{ $labels.instance }}"
+          description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} se llenará en menos de 4 horas."
+
 prometheus:
   prometheusSpec:
     evaluationInterval: 60s