We have the following checks for elasticsearch in Icinga, which will need porting to Prometheus/Alertmanager. Note we do have opensearch alerts per T371083: Port profile::opensearch::monitoring::base_checks to Prometheus/Alertmanager which we can reuse/adapt for some of these too.
define icinga::monitor::elasticsearch::cirrus_checks(
Enum['http', 'https'] $scheme = 'http',
String $host = $::hostname,
Array[Stdlib::Port] $ports = [9200],
Integer $timeout = 4,
) {
$ports.each |$port| {
monitoring::service { "elasticsearch / cirrus frozen writes - ${host}:${port}":
host => $host,
check_command => "check_cirrus_frozen_writes!${scheme}!${port}!${timeout}",
description => "ElasticSearch health check for frozen writes - ${port}",
critical => true,
contact_group => 'admins,team-discovery',
notes_url => 'https://wikitech.wikimedia.org/wiki/Search#Pausing_Indexing',
}
monitoring::service { "elasticsearch / masters eligible - ${host}:${port}":
host => $host,
check_command => "check_masters_eligible!${scheme}!${port}!${timeout}",
description => "ElasticSearch numbers of masters eligible - ${port}",
critical => false,
contact_group => 'admins,team-discovery',
notes_url => 'https://wikitech.wikimedia.org/wiki/Search#Expected_eligible_masters_check_and_alert',
retries => 10, # it is fine if we are missing a master for a short time (during reboots / restarts)
}
}
}# = Define: icinga::monitor::elasticsearch::base_checks
define icinga::monitor::elasticsearch::base_checks(
String $threshold = '>=0.2',
Enum['http', 'https'] $scheme = 'http',
String $host = $::hostname,
Array[Stdlib::Port] $ports = [9200],
Integer $shard_size_warning = 50,
Integer $shard_size_critical = 60,
Integer $timeout = 4,
Boolean $use_nrpe = false,
) {
$ports.each |$port| {
# yes yes! a lot of duplication here which could be improved.
# they will remain here until we find a better way
# also always update both checks!
if !$use_nrpe {
monitoring::service {
default:
host => $host,
critical => false,
contact_group => 'admins,team-discovery',
notes_url => 'https://wikitech.wikimedia.org/wiki/Search#Administration',
;
"elasticsearch_shards_${host}:${port}":
check_command => "check_elasticsearch_shards_threshold!${scheme}!${port}!${threshold}!${timeout}",
description => "ElasticSearch health check for shards on ${port}",
;
"elasticsearch_unassigned_shard_check_${host}:${port}":
check_command => "check_elasticsearch_unassigned_shards!${scheme}!${port}!${timeout}",
description => "ElasticSearch unassigned shard check - ${port}",
check_interval => 720, # 12h
retry_interval => 120, # 2h
retries => 1,
;
"elasticsearch_shard_size_check_${host}:${port}":
check_command => "check_elasticsearch_shard_size!${scheme}!${port}!${shard_size_warning}!${shard_size_critical}!${timeout}",
description => "ElasticSearch shard size check - ${port}",
check_interval => 1440, # 24h
retry_interval => 180, # 3h
notes_url => 'https://wikitech.wikimedia.org/wiki/Search#If_it_has_been_indexed',
;
}
} else {
require ::icinga::elasticsearch::base_plugin
nrpe::monitor_service {
default:
critical => false,
contact_group => 'admins,team-discovery',
notes_url => 'https://wikitech.wikimedia.org/wiki/Search#Administration',
;
"elasticsearch_shards_${port}":
nrpe_command => "/usr/lib/nagios/plugins/check_elasticsearch.py --ignore-status --url http://localhost:${port} --shards-inactive '${threshold}' --timeout ${timeout}",
description => "ElasticSearch health check for shards on ${port}",
;
"elasticsearch_unassigned_shard_check_${port}":
nrpe_command => "/usr/lib/nagios/plugins/check_elasticsearch_unassigned_shards.py --url http://localhost:${port} --timeout ${timeout}",
description => "ElasticSearch unassigned shard check - ${port}",
check_interval => 720, # 12h
retry_interval => 120, # 2h
retries => 1,
;
"elasticsearch_shard_size_check_${port}":
nrpe_command => "/usr/lib/nagios/plugins/check_elasticsearch_shard_size.py --url http://localhost:${port} --shard-size-warning ${shard_size_warning} --shard-size-critical ${shard_size_critical} --timeout ${timeout}",
description => "ElasticSearch shard size check - ${port}",
check_interval => 1440, # 24h
retry_interval => 180, # 3h
notes_url => 'https://wikitech.wikimedia.org/wiki/Search#If_it_has_been_indexed',
;
}
}
}
}define elasticsearch::tlsproxy (
monitoring::service { "elasticsearch-https-${title}":
ensure => present,
description => "Elasticsearch HTTPS for ${title}",
check_command => "${check_command}!${server_name}!${tls_port}",
notes_url => 'https://wikitech.wikimedia.org/wiki/Search',
}! MIGRATION TABLE !
| Migrated? (Y/N) | Title | Resource Type | Command | File | Profiles |
|---|---|---|---|---|---|
| N | elasticsearch_unassigned_shard_check_search.svc.X.wmnet:X | Monitoring::Service | check_elasticsearch_unassigned_shards | modules/icinga/manifests/monitor/elasticsearch/base_checks.pp:17 | profile::icinga |
| N | elasticsearch_shards_search.svc.X.wmnet:X | Monitoring::Service | check_elasticsearch_shards_threshold | modules/icinga/manifests/monitor/elasticsearch/base_checks.pp:17 | profile::icinga |
| N | elasticsearch-https-X | Monitoring::Service | check_ssl_on_port | modules/elasticsearch/manifests/tlsproxy.pp:45 | profile::opensearch::cirrus::server, profile::opensearch::cirrus::relforge |
| N | elasticsearch / masters eligible - search.svc.X.wmnet:X | Monitoring::Service | check_masters_eligible | modules/icinga/manifests/monitor/elasticsearch/cirrus_checks.pp:19 | profile::icinga |
| N | elasticsearch-https-cloudelastic-X-eqiad-X | Monitoring::Service | check_ssl_on_port_letsencrypt | modules/elasticsearch/manifests/tlsproxy.pp:45 | profile::opensearch::cirrus::server |
| N | elasticsearch_shard_size_check_search.svc.X.wmnet:X | Monitoring::Service | check_elasticsearch_shard_size | modules/icinga/manifests/monitor/elasticsearch/base_checks.pp:17 | profile::icinga |