gitops/apps/observability/kube-prom-stack-values.yaml

grafana:
  envFromSecret: "grafana-oidc"
  grafana.ini:
    server:
      root_url: "https://grafana.alexandre-vazquez.cloud"
    auth.generic_oauth:
      enabled: true
      name: Authentik
      icon: signin
      client_id: 7ZYROfeGdeJ6WhmTucgWd3pxzhsIasqpUtWyk5ya
      client_secret: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET}
      scopes: openid email profile
      auth_url: https://authentik.alexandre-vazquez.cloud/application/o/authorize/
      token_url: https://authentik.alexandre-vazquez.cloud/application/o/token/
      api_url: https://authentik.alexandre-vazquez.cloud/application/o/userinfo/
      use_pkce: true
      allow_sign_up: true
      auto_login: false
      role_attribute_path: "contains(groups[*], 'admins') && 'GrafanaAdmin' || 'Viewer'"
      allow_assign_grafana_admin: true
  additionalDataSources:
  - name: loki
    type: loki
    url: http://loki.observability.svc.cluster.local:3100
  ingress:
    enabled: true
    ingressClassName: public
    hosts:
    - grafana.alexandre-vazquez.cloud
    annotations:
      external-dns.alpha.kubernetes.io/hostname: grafana.alexandre-vazquez.internal
  additionalIngresses:
  - name: grafana-ingress-internal
    ingressClassName: public
    hosts:
    - grafana.alexandre-vazquez.internal
    annotations:
      external-dns.alpha.kubernetes.io/hostname: grafana.alexandre-vazquez.internal

kubeApiServer:
  serviceMonitor:
    metricRelabelings:
    - action: drop
      sourceLabels: [__name__]
      regex: apiserver_request_duration_seconds_bucket|apiserver_request_sli_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|apiserver_response_sizes_bucket|apiserver_watch_list_duration_seconds_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_watch_events_sizes_bucket|apiserver_admission_controller_admission_duration_seconds_bucket|apiserver_admission_webhook_admission_duration_seconds_bucket|apiserver_flowcontrol_request_execution_seconds_bucket|apiserver_flowcontrol_request_wait_duration_seconds_bucket

kubeControllerManager:
  endpoints:
  - 192.168.1.40
  serviceMonitor:
    metricRelabelings:
    - action: drop
      sourceLabels: [__name__]
      regex: workqueue_queue_duration_seconds_bucket|workqueue_work_duration_seconds_bucket|storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket

kubeEtcd:
  serviceMonitor:
    metricRelabelings:
    - action: drop
      sourceLabels: [__name__]
      regex: etcd_request_duration_seconds_bucket|etcd_disk_backend_commit_duration_seconds_bucket|etcd_disk_wal_fsync_duration_seconds_bucket|etcd_network_peer_round_trip_time_seconds_bucket

kubeScheduler:
  endpoints:
  - 192.168.1.40
  serviceMonitor:
    metricRelabelings:
    - action: drop
      sourceLabels: [__name__]
      regex: scheduler_plugin_execution_duration_seconds_bucket|scheduler_scheduling_algorithm_duration_seconds_bucket|scheduler_pod_scheduling_duration_seconds_bucket|scheduler_framework_extension_point_duration_seconds_bucket

kubelet:
  serviceMonitor:
    metricRelabelings:
    - action: drop
      sourceLabels: [__name__]
      regex: storage_operation_duration_seconds_bucket|rest_client_request_duration_seconds_bucket|kubelet_runtime_operations_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_bucket

alertmanager:
  config:
    global:
      resolve_timeout: 5m
      smtp_smarthost: '192.168.1.30:587'
      smtp_from: 'alerts@alexandre-vazquez.cloud'
      smtp_auth_username: 'alerts@alexandre-vazquez.cloud'
      smtp_auth_password: '11BE80k1+jbHBirtVGAlAQ=='
      smtp_require_tls: true
    inhibit_rules:
    - equal: [namespace, alertname]
      source_matchers: [severity = critical]
      target_matchers: ['severity =~ warning|info']
    - equal: [namespace, alertname]
      source_matchers: [severity = warning]
      target_matchers: [severity = info]
    - equal: [namespace]
      source_matchers: [alertname = InfoInhibitor]
      target_matchers: [severity = info]
    receivers:
    - name: "null"
    - name: email-alerts
      email_configs:
      - to: 'alexandre.vazquez@gmail.com'
        send_resolved: true
        headers:
          Subject: '[HOMELAB] {{ .GroupLabels.alertname }} — {{ .Status | toUpper }}'
    route:
      group_by: [namespace, alertname]
      group_interval: 5m
      group_wait: 30s
      receiver: "null"
      repeat_interval: 4h
      routes:
      - matchers: ['alertname =~ "InfoInhibitor|Watchdog"']
        receiver: "null"
      - matchers: ['severity =~ "critical|warning"']
        receiver: email-alerts
    templates:
    - /etc/alertmanager/config/*.tmpl

additionalPrometheusRulesMap:
  disk-usage-alerts:
    groups:
    - name: disk.usage
      rules:
      - alert: DiskUsageHigh
        expr: |
          (
            node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
            node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
          ) * 100 < 20
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Disco al {{ printf \"%.0f\" $value }}% libre en {{ $labels.instance }}"
          description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 20% libre."
      - alert: DiskUsageCritical
        expr: |
          (
            node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"} /
            node_filesystem_size_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}
          ) * 100 < 10
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Disco CRÍTICO en {{ $labels.instance }} ({{ printf \"%.0f\" $value }}% libre)"
          description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} tiene menos del 10% libre. ACCIÓN INMEDIATA."
      - alert: DiskWillFillIn4Hours
        expr: |
          predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|fuse.*|squashfs"}[1h], 4*3600) < 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Disco se llenará en <4h en {{ $labels.instance }}"
          description: "Mountpoint {{ $labels.mountpoint }} en {{ $labels.instance }} se llenará en menos de 4 horas."

prometheus:
  prometheusSpec:
    evaluationInterval: 60s
    scrapeInterval: 60s
    storageSpec:
      volumeClaimTemplate:
        spec:
          accessModes:
          - ReadWriteOnce
          storageClassName: ceph-rbd
          resources:
            requests:
              storage: 20Gi