{{- /*
Generated from https://github.com/thanos-io/thanos/blob/main/examples/alerts/alerts.md
*/ -}}
{{- if and .Values.metrics.enabled (or .Values.metrics.prometheusRule.default.create .Values.metrics.prometheusRule.default.receive ) }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: {{ template "common.names.fullname" . }}-receive
  namespace: {{ default .Release.Namespace .Values.metrics.prometheusRule.namespace | quote }}
  labels: {{- include "common.labels.standard" . | nindent 4 }}
    {{- if .Values.metrics.prometheusRule.additionalLabels }}
    {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 4 }}
    {{- end }}
    {{- if .Values.commonLabels }}
    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
    {{- end }}
  {{- if .Values.commonAnnotations }}
  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
  {{- end }}
spec:
  groups:
  - name: thanos-receive
    rules:
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveHttpRequestErrorRateHigh | default false) }}
    - alert: ThanosReceiveHttpRequestErrorRateHigh
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.job {{`}}`}} is failing to handle {{`{{`}} $value | humanize {{`}}`}}% of requests.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehttprequesterrorratehigh
        summary: Thanos Receive is failing to handle requests.
      expr: |
        (
          sum by (job) (rate(http_requests_total{code=~"5..", job=~".*thanos-receive.*", handler="receive"}[5m]))
        /
          sum by (job) (rate(http_requests_total{job=~".*thanos-receive.*", handler="receive"}[5m]))
        ) * 100 > 5
      for: 5m
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveHttpRequestLatencyHigh | default false) }}
    - alert: ThanosReceiveHttpRequestLatencyHigh
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.job {{`}}`}} has a 99th percentile latency of {{`{{`}} $value {{`}}`}} seconds for requests.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehttprequestlatencyhigh
        summary: Thanos Receive has high HTTP requests latency.
      expr: |
        (
          histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~".*thanos-receive.*", handler="receive"}[5m]))) > 10
        and
          sum by (job) (rate(http_request_duration_seconds_count{job=~".*thanos-receive.*", handler="receive"}[5m])) > 0
        )
      for: 10m
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveHighReplicationFailures | default false) }}
    - alert: ThanosReceiveHighReplicationFailures
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.job {{`}}`}} is failing to replicate {{`{{`}} $value | humanize {{`}}`}}% of requests.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighreplicationfailures
        summary: Thanos Receive is having high number of replication failures.
      expr: |
        thanos_receive_replication_factor > 1
          and
        (
          (
            sum by (job) (rate(thanos_receive_replications_total{result="error", job=~".*thanos-receive.*"}[5m]))
          /
            sum by (job) (rate(thanos_receive_replications_total{job=~".*thanos-receive.*"}[5m]))
          )
          >
          (
            max by (job) (floor((thanos_receive_replication_factor{job=~".*thanos-receive.*"}+1) / 2))
          /
            max by (job) (thanos_receive_hashring_nodes{job=~".*thanos-receive.*"})
          )
        ) * 100
      for: 5m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveHighForwardRequestFailures | default false) }}
    - alert: ThanosReceiveHighForwardRequestFailures
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.job {{`}}`}} is failing to forward {{`{{`}} $value | humanize {{`}}`}}% of requests.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighforwardrequestfailures
        summary: Thanos Receive is failing to forward requests.
      expr: |
        (
          sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~".*thanos-receive.*"}[5m]))
        /
          sum by (job) (rate(thanos_receive_forward_requests_total{job=~".*thanos-receive.*"}[5m]))
        ) * 100 > 20
      for: 5m
      labels:
        severity: info
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveHighHashringFileRefreshFailures | default false) }}
    - alert: ThanosReceiveHighHashringFileRefreshFailures
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.job {{`}}`}} is failing to refresh hashring file, {{`{{`}} $value | humanize {{`}}`}} of attempts failed.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures
        summary: Thanos Receive is failing to refresh hasring file.
      expr: |
        (
          sum by (job) (rate(thanos_receive_hashrings_file_errors_total{job=~".*thanos-receive.*"}[5m]))
        /
          sum by (job) (rate(thanos_receive_hashrings_file_refreshes_total{job=~".*thanos-receive.*"}[5m]))
        > 0
        )
      for: 15m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveConfigReloadFailure | default false) }}
    - alert: ThanosReceiveConfigReloadFailure
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.job {{`}}`}} has not been able to reload hashring configurations.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure
        summary: Thanos Receive has not been able to reload configuration.
      expr: avg by (job) (thanos_receive_config_last_reload_successful{job=~".*thanos-receive.*"}) != 1
      for: 5m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveNoUpload | default false) }}
    - alert: ThanosReceiveNoUpload
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Receive {{`{{`}} $labels.instance {{`}}`}} has not uploaded latest data to object storage.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload
        summary: Thanos Receive has not uploaded latest data to object storage.
      expr: |
        (up{job=~".*thanos-receive.*"} - 1)
        + on (job, instance) # filters to only alert on current instance last 3h
        (sum by (job, instance) (increase(thanos_shipper_uploads_total{job=~".*thanos-receive.*"}[3h])) == 0)
      for: 3h
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosReceiveTrafficBelowThreshold | default false) }}
    - alert: ThanosReceiveTrafficBelowThreshold
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: At Thanos Receive {{`{{`}} $labels.job {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} , the average 1-hr avg. metrics ingestion rate  is {{`{{`}} $value | humanize {{`}}`}}% of 12-hr avg. ingestion rate.
        runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivetrafficbelowthreshold
        summary: Thanos Receive is experiencing low avg. 1-hr ingestion rate relative to avg. 12-hr ingestion rate.
      expr: |
        (
          avg_over_time(rate(http_requests_total{job=~".*thanos-receive.*", code=~"2..", handler="receive"}[5m])[1h:5m])
        /
          avg_over_time(rate(http_requests_total{job=~".*thanos-receive.*", code=~"2..", handler="receive"}[5m])[12h:5m])
        ) * 100 < 50
      for: 1h
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
{{- end }}
