elastic-rules.rule

#CPU and memory
ALERT ElasticSearchCpuCritical
  IF es_cpu_percentage > 95
  FOR 3m
  LABELS {severity="critical"}
  ANNOTATIONS {
    description="{{$labels.instance}} reports critical cpu usage. Please verify workload, or add another node to the cluster ",
    summary="Critical CPU usage on {{$labels.instance}}"
  }
# Cluster status
ALERT ElasticSearchStatusCritical
  IF es_status > 1
  FOR 1m
  LABELS {severity="critical"}
  ANNOTATIONS {
    description="{{$labels.instance}} reports critical status of a ElasticSearch cluster {{$labels.cluster}}. Please check additional metrics or logs.",
    summary="Critical cluster status of {{$labels.cluster}} on {{$labels.instance}}"
  }
ALERT ElasticSearchStatusWarning
  IF es_status == 1
  FOR 30m
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="{{$labels.instance}} reports non-healthy status of ElasticSearch cluster {{$labels.cluster}}. Please check additional metrics or logs to find a root cause",
    summary="NonHealthy cluster status of {{$labels.cluster}} on {{$labels.instance}}"
  }

ALERT ElasticSearchUnassigedShards
  IF es_unassigned_shards > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="There are unassigned shards for more than 3 minutes in {{$labels.cluster}} on node {{$labels.instance}}. Please check cluster performance",
    summary="Unassigned shards on {{$labels.cluster}}"
  }

ALERT ElasticSearchActiveShardsPercentage
  IF es_shards_active_percentage < 100
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="Some shards ({{$value}}%) shards are inactive on {{$labels.cluster}} for more than 3 minutes. Results from those shards are unavailable in returned results.",
    summary="Non-active shards on {{$labels.cluster}}"
  }

# Index parameters
ALERT ElasticSearchTooManyIndexFailures
  IF delta(es_indexing_failed_count[1m]) > 0
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="There are documents indexing failures on node {{$labels.instance}}. Please check logs to get more details.",
    summary="Indexing failures on {{$labels.instance}}"
  }

ALERT ElasticSearchIndexIsThrootled
  IF es_indexng_isthrottled > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="Index {{$labels.index}} is throttled for more than 10minutes. Some documents can be missing from returned results.",
    summary="Index {{$labels.index}} throttled for more than 10 minutes"
  }

ALERT ElasticSearchIndexUnassignedShards
  IF es_index_unassigned_shards > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="Unassigned shards on index {{$labels.index}} from {{$labels.cluster}} for more than 3 minutes",
    summary="Unassigned shards on {{$labels.index}} at {{$labels.cluster}}"
  }

ALERT ElasticSearchJvmMemoryPercent
  IF es_jvm_memory_heap_used_percen > 95
  FOR 1m
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="{{$labels.instance}} reports high memory consumption for more than 1 minute. Please check logs for more details",
    summary="High JVM memory consumption on ES node {{$labels.instance}}"
  }

#Cluster settings
ALERT ElasticSearchClusterAllocationDisabled
  IF es_cluster_settings{cluster_routing_allocation_enable="none"} > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
   description="{{$labels.instance}} reports that cluster allocation has been disabled for {{$labels.cluster}}. Some documents can be missing from reported results.",
   summary="Cluster allocation disabled on cluster {{$labels.cluster}}"
  }
ALERT ElasticSearchClusterRebalanceDisabled
  IF es_cluster_settings{cluster_routing_rebalance_enable="none"} > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
   description="{{$labels.instance}} reports that cluster rebalance has been disabled on {{$labels.cluster}}. Some documents can be missing from reported results.",
   summary="Cluster rebalance disabled on cluster {{$labels.cluster}}"
  }

#Cluster settings
ALERT ElasticSearchClusterAllocationDisabledPersistently
  IF es_cluster_persistent_settings{cluster_routing_allocation_enable="none"} > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
   description="{{$labels.instance}} reports that cluster allocation has been disabled persistently for {{$labels.cluster}}. Some documents can be missing from reported results and restart will not help.",
   summary="Cluster allocation disabled on cluster {{$labels.cluster}}"
  }
ALERT ElasticSearchClusterRebalanceDisabledPersistently
  IF es_cluster_persistent_settings{cluster_routing_rebalance_enable="none"} > 0
  FOR 10m
  LABELS {severity="warning"}
  ANNOTATIONS {
   description="{{$labels.instance}} reports that cluster rebalance has been disabled on {{$labels.cluster}}. Some documents can be missing from reported results and restart will not help.",
   summary="Cluster rebalance disabled on cluster {{$labels.cluster}}"
  }

ALERT ElasticSearchClusterReadOnly
  IF es_cluster_settings{cluster_blocks_read_only="true"} > 0
  FOR 1m
  LABELS {severity="critical"}
  ANNOTATIONS {
     description="{{$labels.instance}} reports that cluster {{$labels.cluster}} is in read-only mode. New documents will be rejected.",
     summary="Cluster {{$labels.cluster}} is in RO mode"
    }
ALERT ElasticSearchClusterReadOnlyPersistently
  IF es_cluster_persistent_settings{cluster_blocks_read_only="true"} > 0
  FOR 1m
  LABELS {severity="critical"}
  ANNOTATIONS {
     description="{{$labels.instance}} reports that cluster {{$labels.cluster}} is in read-only mode",
     summary="Cluster {{$labels.cluster}} is read-only"
    }
ALERT ElasticDifferentVersion
  IF count(sum(es_prometheus_version) by (pluginVersion, cluster)) by (cluster) > 1
  FOR 5d
  LABELS {severity="warning"}
  ANNOTATIONS {
     description="Cluster {{$labels.cluster}} reports inconsistent versions of ES plugin",
     summary="Cluster {{$labels.cluster}} is inconsistent"
    }

ALERT ElasticIngestionFailed
  IF delta(es_ingest_total_failed_count[1m]) > 0
  LABELS {severity="warning"}
  ANNOTATIONS {
     description="Node {{$labels.instance}} reports failed ingestions. Some documents were lost",
     summary="Node {{$labels.instance}} ingestion failed"
  }

ALERT ElasticNoSpaceWithin24h
    IF predict_linear(es_fs_free_bytes[1h], 24*3600) < 0
    FOR 10m
    LABELS {severity="warning"}
    ANNOTATIONS {
      description="Elasticsearch reports that space on {{ $labels.node }}({{ $labels.instance }}) will run within 24h. Please check disk usage on that host",
      summary="Elasticsearch {{ $labels.node }} at {{ $labels.cluster }} will be out of disk space within 24h"
  }

ALERT ElasticNoAvailableSpace
  IF es_fs_path_free_bytes * 100 / es_fs_path_total_bytes < 10
  FOR 10m
  LABELS {severity="critical"}
  ANNOTATIONS {
    description="Elasticsearch reports that there  are only {{ $value }}% left on {{ $labels.path }} at {{$labels.instance}}. Please check it",
    summary="No Available space on {{$labels.instance}}"
  }

ALERT ElasticCircuitEnabled
  IF delta(es_breaker_tripped[1m]) > 0
  LABELS {severity="critical"}
  ANNOTATIONS {
    description="Elasticsearch circuit breaker {{ $labels.circuit_name }} was enabled within last minute on {{$labels.node}} ({{$labels.instance}}). Looks like high memory pressure on this host ans some data were discarded.",
    summary="Circuit breaker {{ $labels.circuit_name}} enabled on {{$labels.node}}"
  }

ALERT ElasticThreadpoolRejected
  IF delta(es_threadpool_rejected[1m]) > 0
  LABELS {severity="warning"}
  ANNOTATIONS {
    description="Elasticsearch threadpool {{ $labels.threadpool }} rejected {{ $value }} tasks within last minute. Some jobs failed and never will be repeated, it could be a high CPU pressure or I/O errors. Please check node and it's state",
    summary="Threadpool tasks rejected {{ $labels.threadpool}} at {{$labels.node}}"
  }