Event Timeline
Comment Actions
elastic1066 is showing far more rejections that the rest.
The list of shards served from elastic1066 is:
afwiki_general_1728039174 0 p STARTED 243090 1.2gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.01.01 0 r STARTED 39357829 2gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.01.17 0 r STARTED 34828683 1.9gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.01.27 0 r STARTED 34718591 1.9gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.01.31 0 r STARTED 29405670 1.7gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.02.05 0 r STARTED 29649811 1.7gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.02.10 0 r STARTED 26709964 1.5gb 10.64.48.136 elastic1066-production-search-eqiad apifeatureusage-2025.02.12 0 r STARTED 24441264 1.4gb 10.64.48.136 elastic1066-production-search-eqiad arwiki_content_1728056277 0 p STARTED 358365 6.9gb 10.64.48.136 elastic1066-production-search-eqiad arwiki_general_1728096140 0 p STARTED 1578436 9.8gb 10.64.48.136 elastic1066-production-search-eqiad azbwiki_general_1728074141 0 p STARTED 297186 593.8mb 10.64.48.136 elastic1066-production-search-eqiad bewiki_general_1728060641 0 r STARTED 177498 1gb 10.64.48.136 elastic1066-production-search-eqiad bgwiki_archive_1717279079 0 r STARTED 71678 4.9mb 10.64.48.136 elastic1066-production-search-eqiad bswiki_archive_1717269969 0 r STARTED 84569 3.9mb 10.64.48.136 elastic1066-production-search-eqiad commonswiki_file_1717000466 11 p STARTED 4442923 32.3gb 10.64.48.136 elastic1066-production-search-eqiad commonswiki_file_1727809582 18 r STARTED 4862864 52.7gb 10.64.48.136 elastic1066-production-search-eqiad commonswiki_general_1727903574 7 r STARTED 2056718 19.4gb 10.64.48.136 elastic1066-production-search-eqiad commonswiki_titlesuggest_1709176056 0 r STARTED 203267 55.5mb 10.64.48.136 elastic1066-production-search-eqiad cywiki_general_1728129378 0 p STARTED 169610 650.2mb 10.64.48.136 elastic1066-production-search-eqiad dewiki_content_1727860262 7 r STARTED 573442 10.8gb 10.64.48.136 elastic1066-production-search-eqiad enwiki_content_1728079212 1 r STARTED 628783 20.1gb 10.64.48.136 elastic1066-production-search-eqiad enwiki_general_1727860262 16 r STARTED 1974243 27.2gb 10.64.48.136 elastic1066-production-search-eqiad enwikisource_archive_1717279608 0 r STARTED 95665 6.5mb 10.64.48.136 elastic1066-production-search-eqiad enwiki_titlesuggest_1740972983 2 r STARTED 2571305 931.8mb 10.64.48.136 elastic1066-production-search-eqiad fiwiki_content_1728102665 0 r STARTED 223680 4.9gb 10.64.48.136 elastic1066-production-search-eqiad frwiki_content_1727958460 1 r STARTED 464406 15.5gb 10.64.48.136 elastic1066-production-search-eqiad frwiki_general_1727890968 0 r STARTED 1091279 12.1gb 10.64.48.136 elastic1066-production-search-eqiad frwiktionary_general_1727932638 0 r STARTED 743390 3.8gb 10.64.48.136 elastic1066-production-search-eqiad glent_20230422 0 r STARTED 38470320 2.1gb 10.64.48.136 elastic1066-production-search-eqiad glent_20231216 0 r STARTED 42386027 2.6gb 10.64.48.136 elastic1066-production-search-eqiad glent_20240309 1 r STARTED 43815148 2.4gb 10.64.48.136 elastic1066-production-search-eqiad glent_20240406 0 r STARTED 44500744 2.5gb 10.64.48.136 elastic1066-production-search-eqiad glent_20240518 1 r STARTED 45269679 2.7gb 10.64.48.136 elastic1066-production-search-eqiad glent_20240608 0 r STARTED 45566957 2.5gb 10.64.48.136 elastic1066-production-search-eqiad glent_20240629 1 r STARTED 45858759 3gb 10.64.48.136 elastic1066-production-search-eqiad hewikisource_archive_1717245858 0 r STARTED 63534 3.1mb 10.64.48.136 elastic1066-production-search-eqiad hrwiki_archive_1717405558 0 r STARTED 186454 9.7mb 10.64.48.136 elastic1066-production-search-eqiad huwiktionary_archive_1717328155 0 r STARTED 4648 361.8kb 10.64.48.136 elastic1066-production-search-eqiad hywikisource_archive_1717316836 0 r STARTED 2403 259.1kb 10.64.48.136 elastic1066-production-search-eqiad idwiki_titlesuggest_1740980543 0 p STARTED 854559 261.5mb 10.64.48.136 elastic1066-production-search-eqiad iowiktionary_content_1728117860 0 p STARTED 329847 784.8mb 10.64.48.136 elastic1066-production-search-eqiad iswiki_general_1727965990 0 p STARTED 57611 307.3mb 10.64.48.136 elastic1066-production-search-eqiad itwikibooks_archive_1717346073 0 p STARTED 3830 422kb 10.64.48.136 elastic1066-production-search-eqiad itwiki_content_1727938747 3 r STARTED 328209 8.2gb 10.64.48.136 elastic1066-production-search-eqiad itwiki_general_1727868002 7 r STARTED 655697 6.2gb 10.64.48.136 elastic1066-production-search-eqiad itwikiquote_general_1727935063 0 r STARTED 114893 437.4mb 10.64.48.136 elastic1066-production-search-eqiad itwikiquote_titlesuggest_1724304032 0 p STARTED 64018 16.7mb 10.64.48.136 elastic1066-production-search-eqiad itwikisource_content_1727874287 0 p STARTED 351448 4.4gb 10.64.48.136 elastic1066-production-search-eqiad itwiki_titlesuggest_1740981166 1 r STARTED 1203870 307.8mb 10.64.48.136 elastic1066-production-search-eqiad jawiki_content_1727985243 1 r STARTED 254582 11.6gb 10.64.48.136 elastic1066-production-search-eqiad jawikisource_titlesuggest_1674708973 0 r STARTED 17840 8.3mb 10.64.48.136 elastic1066-production-search-eqiad jvwiki_content_1728049976 0 r STARTED 90409 1001.1mb 10.64.48.136 elastic1066-production-search-eqiad knwikisource_general_1727930176 0 p STARTED 6310 81mb 10.64.48.136 elastic1066-production-search-eqiad kowikisource_content_1728064474 0 r STARTED 58089 4.2gb 10.64.48.136 elastic1066-production-search-eqiad ltwiktionary_archive_1717247749 0 r STARTED 13917 656.1kb 10.64.48.136 elastic1066-production-search-eqiad lvwiki_titlesuggest_1709271830 0 r STARTED 201466 54.6mb 10.64.48.136 elastic1066-production-search-eqiad metawiki_archive_1717251387 0 p STARTED 292805 17mb 10.64.48.136 elastic1066-production-search-eqiad metawiki_general_1727872329 2 r STARTED 1534778 10.8gb 10.64.48.136 elastic1066-production-search-eqiad mgwiktionary_content_1728095592 3 p STARTED 1268887 2.4gb 10.64.48.136 elastic1066-production-search-eqiad mrwiki_general_1728133488 0 r STARTED 134608 1.2gb 10.64.48.136 elastic1066-production-search-eqiad ndswiki_archive_1717334088 0 r STARTED 1654 147kb 10.64.48.136 elastic1066-production-search-eqiad newwiki_archive_1717201212 0 p STARTED 33368 1.2mb 10.64.48.136 elastic1066-production-search-eqiad nowiki_content_1728107303 1 r STARTED 264834 3.5gb 10.64.48.136 elastic1066-production-search-eqiad nowiki_titlesuggest_1740984421 0 r STARTED 809751 196.6mb 10.64.48.136 elastic1066-production-search-eqiad ptwiki_content_1728003051 4 r STARTED 201176 5.1gb 10.64.48.136 elastic1066-production-search-eqiad rowiki_content_1728106428 0 r STARTED 259695 4.6gb 10.64.48.136 elastic1066-production-search-eqiad rowikisource_titlesuggest_1709188999 0 p STARTED 16395 7.9mb 10.64.48.136 elastic1066-production-search-eqiad ruwiki_general_1727923233 6 r STARTED 413843 3.5gb 10.64.48.136 elastic1066-production-search-eqiad ruwikinews_content_1728098112 1 p STARTED 374687 10gb 10.64.48.136 elastic1066-production-search-eqiad ruwiki_titlesuggest_1740986451 0 r STARTED 1012384 441.4mb 10.64.48.136 elastic1066-production-search-eqiad sawiktionary_content_1728068508 0 r STARTED 262225 2.3gb 10.64.48.136 elastic1066-production-search-eqiad slwikisource_archive_1717308854 0 p STARTED 2257 235.4kb 10.64.48.136 elastic1066-production-search-eqiad sqwiki_general_1728130624 0 r STARTED 152042 1gb 10.64.48.136 elastic1066-production-search-eqiad svwiki_content_1727953317 6 r STARTED 588268 5.7gb 10.64.48.136 elastic1066-production-search-eqiad thwiki_archive_1717218947 0 r STARTED 170664 12mb 10.64.48.136 elastic1066-production-search-eqiad thwikibooks_archive_1717316543 0 p STARTED 6368 489.4kb 10.64.48.136 elastic1066-production-search-eqiad thwiki_general_1727955003 0 r STARTED 724191 4.3gb 10.64.48.136 elastic1066-production-search-eqiad ukwiki_content_1727946044 3 r STARTED 335742 5.1gb 10.64.48.136 elastic1066-production-search-eqiad viwiki_general_1728073492 0 r STARTED 3557061 23.8gb 10.64.48.136 elastic1066-production-search-eqiad vowiki_content_1728063857 0 r STARTED 40495 172.9mb 10.64.48.136 elastic1066-production-search-eqiad wikidatawiki_content_1727882536 17 p STARTED 5963852 31.4gb 10.64.48.136 elastic1066-production-search-eqiad wikidatawiki_content_1728009803 8 r STARTED 6076275 42gb 10.64.48.136 elastic1066-production-search-eqiad wikidatawiki_general_1728093363 0 r STARTED 674518 25.2gb 10.64.48.136 elastic1066-production-search-eqiad
It has some hot shards like:
- enwiki_content
- enwiki_titlesuggest
- dewiki_content
- frwiki_content
Shard search stats for enwiki_content@1 is:
curl -s localhost:9200/_stats/search?level=shards | jq '.indices.enwiki_content_1728079212.shards."1"'
[ { "routing": { "state": "STARTED", "primary": true, "node": "FovTizPSSpaKmBzxoCJpmQ", "relocating_node": null }, "search": { "open_contexts": 3, "query_total": 26194447, "query_time_in_millis": 1335665831, "query_current": 1, "fetch_total": 7694799, "fetch_time_in_millis": 75924331, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "commit": { "id": "CLZFsWF55Q2ox+0PwCP8UQ==", "generation": 903, "user_data": { "local_checkpoint": "9610978", "max_unsafe_auto_id_timestamp": "-1", "min_retained_seq_no": "9610103", "translog_uuid": "7zPoTrr0TaS3RctO6repvg", "history_uuid": "4WeVVHR8Tl2J1uMIrQ2HaQ", "max_seq_no": "9610978" }, "num_docs": 628782 }, "seq_no": { "max_seq_no": 9613187, "local_checkpoint": 9613187, "global_checkpoint": 9613187 }, "retention_leases": { "primary_term": 3, "version": 1228576, "leases": [ { "id": "peer_recovery/FovTizPSSpaKmBzxoCJpmQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" }, { "id": "peer_recovery/ocZjIxEhQb-TojSu1v6NYQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" }, { "id": "peer_recovery/eS9G_qj7RlC5dFdhH12MMQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" } ] }, "shard_path": { "state_path": "/srv/elasticsearch/production-search-eqiad/nodes/0", "data_path": "/srv/elasticsearch/production-search-eqiad/nodes/0", "is_custom_data_path": false } }, { "routing": { "state": "STARTED", "primary": false, "node": "ocZjIxEhQb-TojSu1v6NYQ", "relocating_node": null }, "search": { "open_contexts": 3, "query_total": 29366891, "query_time_in_millis": 1415151460, "query_current": 2, "fetch_total": 8706023, "fetch_time_in_millis": 83860472, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "commit": { "id": "LOmO9FFutdoPNzAIqdNGbw==", "generation": 907, "user_data": { "local_checkpoint": "9600049", "max_unsafe_auto_id_timestamp": "-1", "min_retained_seq_no": "9597758", "translog_uuid": "a44BURyfSVm5IuYKpDZTJQ", "history_uuid": "4WeVVHR8Tl2J1uMIrQ2HaQ", "max_seq_no": "9600049" }, "num_docs": 628777 }, "seq_no": { "max_seq_no": 9613187, "local_checkpoint": 9613187, "global_checkpoint": 9613187 }, "retention_leases": { "primary_term": 3, "version": 1228576, "leases": [ { "id": "peer_recovery/FovTizPSSpaKmBzxoCJpmQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" }, { "id": "peer_recovery/ocZjIxEhQb-TojSu1v6NYQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" }, { "id": "peer_recovery/eS9G_qj7RlC5dFdhH12MMQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" } ] }, "shard_path": { "state_path": "/srv/elasticsearch/production-search-eqiad/nodes/0", "data_path": "/srv/elasticsearch/production-search-eqiad/nodes/0", "is_custom_data_path": false } }, { "routing": { "state": "STARTED", "primary": false, "node": "eS9G_qj7RlC5dFdhH12MMQ", "relocating_node": null }, "search": { "open_contexts": 4, "query_total": 29269774, "query_time_in_millis": 1885608565, "query_current": 1, "fetch_total": 8834836, "fetch_time_in_millis": 106161584, "fetch_current": 0, "scroll_total": 0, "scroll_time_in_millis": 0, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "commit": { "id": "xUeygehX0MfXHMfrGyCJ/g==", "generation": 910, "user_data": { "local_checkpoint": "9611743", "max_unsafe_auto_id_timestamp": "-1", "min_retained_seq_no": "9592030", "translog_uuid": "aajb9KvgQ_yNPKiJKlVztw", "history_uuid": "4WeVVHR8Tl2J1uMIrQ2HaQ", "max_seq_no": "9611749" }, "num_docs": 628783 }, "seq_no": { "max_seq_no": 9613187, "local_checkpoint": 9613187, "global_checkpoint": 9613187 }, "retention_leases": { "primary_term": 3, "version": 1228576, "leases": [ { "id": "peer_recovery/FovTizPSSpaKmBzxoCJpmQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" }, { "id": "peer_recovery/ocZjIxEhQb-TojSu1v6NYQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" }, { "id": "peer_recovery/eS9G_qj7RlC5dFdhH12MMQ", "retaining_seq_no": 9613186, "timestamp": 1740993196111, "source": "peer recovery" } ] }, "shard_path": { "state_path": "/srv/elasticsearch/production-search-eqiad/nodes/0", "data_path": "/srv/elasticsearch/production-search-eqiad/nodes/0", "is_custom_data_path": false } } ]