{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}

{{- if and .Values.prometheus.enabled .Values.prometheus.defaultRules.create .Values.prometheus.defaultRules.rules.nodeExporterAlerting }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: {{ printf "%s-node-exporter" (include "kube-prometheus.prometheus.fullname" .) }}
  namespace: {{ include "common.names.namespace" . | quote }}
  labels: {{ include "kube-prometheus.prometheus.labels" . | nindent 4 }}
  {{- if .Values.commonAnnotations }}
  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" . ) | nindent 4 }}
  {{- end }}
spec:
  groups:
    - name: node-exporter
      rules:
        - alert: NodeFilesystemSpaceFillingUp
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
            summary: Filesystem is predicted to run out of space within the next 24 hours.
          expr: |-
            (
              node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15
            and
              predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 1h
          labels:
            severity: warning
        - alert: NodeFilesystemSpaceFillingUp
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
            summary: Filesystem is predicted to run out of space within the next 4 hours.
          expr: |-
            (
              node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10
            and
              predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 1h
          labels:
            severity: critical
        - alert: NodeFilesystemAlmostOutOfSpace
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
            summary: Filesystem has less than 5% space left.
          expr: |-
            (
              node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 30m
          labels:
            severity: warning
        - alert: NodeFilesystemAlmostOutOfSpace
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
            summary: Filesystem has less than 3% space left.
          expr: |-
            (
              node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 30m
          labels:
            severity: critical
        - alert: NodeFilesystemFilesFillingUp
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
            summary: Filesystem is predicted to run out of inodes within the next 24 hours.
          expr: |-
            (
              node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40
            and
              predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 1h
          labels:
            severity: warning
        - alert: NodeFilesystemFilesFillingUp
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
            summary: Filesystem is predicted to run out of inodes within the next 4 hours.
          expr: |-
            (
              node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20
            and
              predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 1h
          labels:
            severity: critical
        - alert: NodeFilesystemAlmostOutOfFiles
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
            summary: Filesystem has less than 5% inodes left.
          expr: |-
            (
              node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 1h
          labels:
            severity: warning
        - alert: NodeFilesystemAlmostOutOfFiles
          annotations:
            description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
            summary: Filesystem has less than 3% inodes left.
          expr: |-
            (
              node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
            and
              node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
            )
          for: 1h
          labels:
            severity: critical
        - alert: NodeNetworkReceiveErrs
          annotations:
            description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.'
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
            summary: Network interface is reporting many receive errors.
          expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01
          for: 1h
          labels:
            severity: warning
        - alert: NodeNetworkTransmitErrs
          annotations:
            description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.'
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
            summary: Network interface is reporting many transmit errors.
          expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01
          for: 1h
          labels:
            severity: warning
        - alert: NodeHighNumberConntrackEntriesUsed
          annotations:
            description: '{{`{{`}} $labels.instance {{`}}`}} {{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.'
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
            summary: Number of conntrack are getting close to the limit.
          expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75
          labels:
            severity: warning
        - alert: NodeTextFileCollectorScrapeError
          annotations:
            description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
            summary: Node Exporter text file collector failed to scrape.
          expr: node_textfile_scrape_error{job="node-exporter"} == 1
          labels:
            severity: warning
        - alert: NodeClockSkewDetected
          annotations:
            description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
            summary: Clock skew detected.
          expr: |-
            (
              node_timex_offset_seconds{job="node-exporter"} > 0.05
            and
              deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
            )
            or
            (
              node_timex_offset_seconds{job="node-exporter"} < -0.05
            and
              deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
            )
          for: 10m
          labels:
            severity: warning
        - alert: NodeClockNotSynchronising
          annotations:
            description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
            summary: Clock not synchronising.
          expr: |-
            min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
            and
            node_timex_maxerror_seconds{job="node-exporter"} >= 16
          for: 10m
          labels:
            severity: warning
        - alert: NodeRAIDDegraded
          annotations:
            description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
            summary: RAID Array is degraded.
          expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0
          for: 15m
          labels:
            severity: critical
        - alert: NodeRAIDDiskFailure
          annotations:
            description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
            summary: Failed device in RAID array.
          expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0
          labels:
            severity: warning
        - alert: NodeFileDescriptorLimit
          annotations:
            description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
            summary: Kernel is predicted to exhaust file descriptors limit soon.
          expr: |-
            (
              node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
            )
          for: 15m
          labels:
            severity: warning
        - alert: NodeFileDescriptorLimit
          annotations:
            description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
            summary: Kernel is predicted to exhaust file descriptors limit soon.
          expr: |-
            (
              node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
            )
          for: 15m
          labels:
            severity: critical
        - alert: NodeCPUHighUsage
          annotations:
            description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. '
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
            summary: High CPU usage.
          expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!~"idle|iowait"}[2m]))) * 100 > 90
          for: 15m
          labels:
            severity: info
        - alert: NodeSystemSaturation
          annotations:
            description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. This might indicate this instance resources saturation and can cause it becoming unresponsive. '
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
            summary: System saturated, load per core is very high.
          expr: |-
            node_load1{job="node-exporter"}
            / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2
          for: 15m
          labels:
            severity: warning
        - alert: NodeMemoryMajorPagesFaults
          annotations:
            description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. Please check that there is enough memory available at this instance. '
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
            summary: Memory major page faults are occurring at very high rate.
          expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
          for: 15m
          labels:
            severity: warning
        - alert: NodeMemoryHighUtilization
          annotations:
            description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. '
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
            summary: Host is running out of memory.
          expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90
          for: 15m
          labels:
            severity: warning
        - alert: NodeDiskIOSaturation
          annotations:
            description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. This symptom might indicate disk saturation. '
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
            summary: Disk IO queue is high.
          expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10
          for: 30m
          labels:
            severity: warning
        - alert: NodeSystemdServiceFailed
          annotations:
            description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}}
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
            summary: Systemd service has entered failed state.
          expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
          for: 5m
          labels:
            severity: warning
        - alert: NodeSystemdServiceCrashlooping
          annotations:
            description: Systemd service {{`{{`}} $labels.name {{`}}`}} has being restarted too many times at {{`{{`}} $labels.instance {{`}}`}} for the last 15 minutes. Please check if service is crash looping.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicecrashlooping
            summary: Systemd service keeps restaring, possibly crash looping.
          expr: increase(node_systemd_service_restart_total{job="node-exporter"}[5m]) > 2
          for: 15m
          labels:
            severity: warning
        - alert: NodeBondingDegraded
          annotations:
            description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures.
            runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
            summary: Bonding interface is degraded
          expr: (node_bonding_slaves - node_bonding_active) != 0
          for: 5m
          labels:
            severity: warning
{{- end }}
