{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}

{{- if and .Values.prometheus.enabled .Values.prometheus.defaultRules.create .Values.prometheus.defaultRules.rules.etcd }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: {{ printf "%s-etcd" (include "kube-prometheus.prometheus.fullname" .) }}
  namespace: {{ include "common.names.namespace" . | quote }}
  labels: {{ include "kube-prometheus.prometheus.labels" . | nindent 4 }}
  {{- if .Values.commonAnnotations }}
  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" . ) | nindent 4 }}
  {{- end }}
spec:
  groups:
    - name: etcd
      rules:
        - alert: etcdMembersDown
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).'
            summary: etcd cluster members are down.
          expr: |-
            max without (endpoint) (
              sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
            or
              count without (To) (
                sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
              )
            )
            > 0
          for: 20m
          labels:
            severity: warning
        - alert: etcdInsufficientMembers
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).'
            summary: etcd cluster has insufficient number of members.
          expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance, pod) < ((count(up{job=~".*etcd.*"}) without (instance, pod) + 1) / 2)
          for: 3m
          labels:
            severity: critical
        - alert: etcdNoLeader
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.'
            summary: etcd cluster has no leader.
          expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
          for: 1m
          labels:
            severity: critical
        - alert: etcdHighNumberOfLeaderChanges
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
            summary: etcd cluster has high number of leader changes.
          expr: increase((max without (instance, pod) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
          for: 5m
          labels:
            severity: warning
        - alert: etcdHighNumberOfFailedGRPCRequests
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster has high number of failed grpc requests.
          expr: |-
            100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
              /
            sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
              > 1
          for: 10m
          labels:
            severity: warning
        - alert: etcdHighNumberOfFailedGRPCRequests
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster has high number of failed grpc requests.
          expr: |-
            100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
              /
            sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
              > 5
          for: 5m
          labels:
            severity: critical
        - alert: etcdGRPCRequestsSlow
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.'
            summary: etcd grpc requests are slow
          expr: |-
            histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type))
            > 0.15
          for: 10m
          labels:
            severity: critical
        - alert: etcdMemberCommunicationSlow
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster member communication is slow.
          expr: |-
            histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
            > 0.15
          for: 10m
          labels:
            severity: warning
        - alert: etcdHighNumberOfFailedProposals
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster has high number of proposal failures.
          expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
          for: 15m
          labels:
            severity: warning
        - alert: etcdHighFsyncDurations
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster 99th percentile fsync durations are too high.
          expr: |-
            histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
            > 0.5
          for: 10m
          labels:
            severity: warning
        - alert: etcdHighFsyncDurations
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster 99th percentile fsync durations are too high.
          expr: |-
            histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
            > 1
          for: 10m
          labels:
            severity: critical
        - alert: etcdHighCommitDurations
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
            summary: etcd cluster 99th percentile commit durations are too high.
          expr: |-
            histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
            > 0.25
          for: 10m
          labels:
            severity: warning
        - alert: etcdDatabaseQuotaLowSpace
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
            summary: etcd cluster database is running full.
          expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95
          for: 10m
          labels:
            severity: critical
        - alert: etcdExcessiveDatabaseGrowth
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.'
            summary: etcd cluster database growing very fast.
          expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
          for: 10m
          labels:
            severity: warning
        - alert: etcdDatabaseHighFragmentationRatio
          annotations:
            description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
            runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
            summary: etcd database size in use is less than 50% of the actual allocated storage.
          expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600
          for: 10m
          labels:
            severity: warning
{{- end }}
