Page MenuHomePhabricator
Paste P73928

search thread pool (production-search-eqiad 2025-03-03T09:00)
ActivePublic

Authored by dcausse on Mar 3 2025, 8:59 AM.
Project Tags
None
Referenced Files
Subscribers
None
curl 'localhost:9200/_cat/thread_pool/search?h=node_name,node_id,n,l,type,completed,rejected&v'
node_name node_id n l type completed rejected
elastic1071-production-search-eqiad ocZjIxEhQb-TojSu1v6NYQ search 73 fixed_auto_queue_size 140760721 0
elastic1056-production-search-eqiad 8grTo-qVTe-a-QOptlXvAA search 61 fixed_auto_queue_size 79014359 0
elastic1066-production-search-eqiad eS9G_qj7RlC5dFdhH12MMQ search 61 fixed_auto_queue_size 142933677 66406
elastic1075-production-search-eqiad skuKHcd2RiiQvO7rWy0UNw search 73 fixed_auto_queue_size 130373145 0
elastic1079-production-search-eqiad VRrsJJr7RcCF6S-e5GgCnw search 73 fixed_auto_queue_size 95059824 0
elastic1053-production-search-eqiad bExyt2APQj6SeedQ8nM_kg search 61 fixed_auto_queue_size 114400764 14
elastic1082-production-search-eqiad 0vVqY9ADSYGwbm_ARIsgNA search 73 fixed_auto_queue_size 93178339 0
elastic1093-production-search-eqiad zBp5OH5tSjqFXxiFqr5wTQ search 73 fixed_auto_queue_size 117020740 0
elastic1099-production-search-eqiad qCnaIqurQg-EK2rnzv5eMg search 73 fixed_auto_queue_size 105590995 227
elastic1101-production-search-eqiad MJTXVXN8QV-2C7dQzmkXaQ search 73 fixed_auto_queue_size 99190428 0
elastic1081-production-search-eqiad W_dyTjX-QQ-3ebNItdDd7A search 73 fixed_auto_queue_size 69375294 0
elastic1092-production-search-eqiad BPyEn_O1RGuMo7dl0D3pGg search 73 fixed_auto_queue_size 157631432 0
elastic1064-production-search-eqiad O7qQYZ1wQdOr7CCXyYCL2Q search 61 fixed_auto_queue_size 128263951 150
elastic1057-production-search-eqiad kgFgbTIXSk2rMD4xdbwdzQ search 61 fixed_auto_queue_size 122642473 0
elastic1077-production-search-eqiad TVaQni9_RMCgwKGMCxC8oA search 73 fixed_auto_queue_size 131084035 0
elastic1078-production-search-eqiad mVJGnmtoTECCmeYCOY1TQg search 73 fixed_auto_queue_size 94723965 0
elastic1100-production-search-eqiad 3kTB612jRjWcsXsCa7n3ag search 73 fixed_auto_queue_size 95435475 0
elastic1055-production-search-eqiad I9qt5dmVQWSuHnyHVNTy8w search 61 fixed_auto_queue_size 102787760 0
elastic1068-production-search-eqiad HVkrQofuREeD6z68wYpbRw search 73 fixed_auto_queue_size 137317544 0
elastic1087-production-search-eqiad wfEUsWcVSvaXnJipLotpTg search 73 fixed_auto_queue_size 125508045 0
elastic1076-production-search-eqiad GvYM3SR1T6mSNOIMzozbAg search 73 fixed_auto_queue_size 158759125 244
elastic1088-production-search-eqiad HBYEO1I6SRCOBpPI2vf0VQ search 73 fixed_auto_queue_size 74103683 0
elastic1080-production-search-eqiad RpvFzrRQRtK_pMRJKiTwqA search 73 fixed_auto_queue_size 111207258 0
elastic1059-production-search-eqiad mQjEylJUQnOQHPYLRdaZLQ search 61 fixed_auto_queue_size 113066763 0
elastic1102-production-search-eqiad k2C5cK9gSVuSFdVAp_qFFg search 73 fixed_auto_queue_size 97447512 0
elastic1103-production-search-eqiad bmrbrH5ZTSGN2Fbok2d7zw search 73 fixed_auto_queue_size 82422291 0
elastic1072-production-search-eqiad p_uTa9F_S1ahW4H3vPhkng search 73 fixed_auto_queue_size 119002741 0
elastic1090-production-search-eqiad FovTizPSSpaKmBzxoCJpmQ search 73 fixed_auto_queue_size 143121590 0
elastic1095-production-search-eqiad I6p3CXx9R8-W7imJI6o3qA search 73 fixed_auto_queue_size 110689737 0
elastic1070-production-search-eqiad r5WL00MqTma0O3076t_gGA search 73 fixed_auto_queue_size 106247778 0
elastic1085-production-search-eqiad pQ2bYcg8Q6K3b13LCkethA search 73 fixed_auto_queue_size 121643107 0
elastic1073-production-search-eqiad kWoKB6-nSg2iCs8nyAfoHA search 73 fixed_auto_queue_size 67612983 0
elastic1098-production-search-eqiad asamIuHMTY2yLO1lEspsKg search 73 fixed_auto_queue_size 96457998 0
elastic1089-production-search-eqiad VHlgUshsTPSaKND6yBjzLA search 73 fixed_auto_queue_size 120917203 0
elastic1065-production-search-eqiad dyeVcZz_QNiq2S01FK2oEA search 61 fixed_auto_queue_size 97168607 0
elastic1094-production-search-eqiad OzWJnHZsQSai3JDms89bTw search 73 fixed_auto_queue_size 86060513 0
elastic1067-production-search-eqiad 4spO9PA_Qka_vQglCASEig search 61 fixed_auto_queue_size 121257558 53
elastic1061-production-search-eqiad zjt3S6fWRzS1VIRsnYQHoA search 61 fixed_auto_queue_size 101177955 0
elastic1074-production-search-eqiad xH6aV6uRTsGgttDYrZg-cw search 73 fixed_auto_queue_size 122820640 349
elastic1097-production-search-eqiad 3Ak-3kHcRYGHunk2DqKHJA search 73 fixed_auto_queue_size 146504887 0
elastic1086-production-search-eqiad 08kfpaHFTEudOoNK5junHg search 73 fixed_auto_queue_size 154274966 0
elastic1060-production-search-eqiad pdSgXpRMRgKUPWPbL7n9iA search 61 fixed_auto_queue_size 167855363 203
elastic1096-production-search-eqiad vIB_GTFSSjKJ6S9gcgUO7A search 73 fixed_auto_queue_size 148264725 0
elastic1062-production-search-eqiad 4WHfBZy3R0qnS2VKiRCVFQ search 61 fixed_auto_queue_size 142973628 18
elastic1084-production-search-eqiad Dkqrjm6iTg-jLFj2suXV9w search 73 fixed_auto_queue_size 129601076 0
elastic1107-production-search-eqiad V5M4_rsRSbqRy8Nna4-QVw search 73 fixed_auto_queue_size 59469869 0
elastic1083-production-search-eqiad Wq9YyhClTr-mrQ_BhLw-xw search 73 fixed_auto_queue_size 33724941 0
elastic1054-production-search-eqiad X4qotYsJRp-DdkoalYLDEg search 61 fixed_auto_queue_size 64839243 0
elastic1058-production-search-eqiad S9bCRsKqTTiod9tlh_DG_A search 61 fixed_auto_queue_size 141341553 0
elastic1063-production-search-eqiad nhAXB8drQ-qvYOmpMnaESg search 61 fixed_auto_queue_size 107602831 0
elastic1091-production-search-eqiad wH588zgNTe6dQffXXQ3u0A search 73 fixed_auto_queue_size 123051257 0
elastic1069-production-search-eqiad 9TaKBtNwRuiVBCAVBGhH7w search 73 fixed_auto_queue_size 134744099 0

Event Timeline

elastic1066 is showing far more rejections that the rest.

The list of shards served from elastic1066 is:

afwiki_general_1728039174              0  p STARTED        243090    1.2gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.01.01             0  r STARTED      39357829      2gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.01.17             0  r STARTED      34828683    1.9gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.01.27             0  r STARTED      34718591    1.9gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.01.31             0  r STARTED      29405670    1.7gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.02.05             0  r STARTED      29649811    1.7gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.02.10             0  r STARTED      26709964    1.5gb 10.64.48.136 elastic1066-production-search-eqiad
apifeatureusage-2025.02.12             0  r STARTED      24441264    1.4gb 10.64.48.136 elastic1066-production-search-eqiad
arwiki_content_1728056277              0  p STARTED        358365    6.9gb 10.64.48.136 elastic1066-production-search-eqiad
arwiki_general_1728096140              0  p STARTED       1578436    9.8gb 10.64.48.136 elastic1066-production-search-eqiad
azbwiki_general_1728074141             0  p STARTED        297186  593.8mb 10.64.48.136 elastic1066-production-search-eqiad
bewiki_general_1728060641              0  r STARTED        177498      1gb 10.64.48.136 elastic1066-production-search-eqiad
bgwiki_archive_1717279079              0  r STARTED         71678    4.9mb 10.64.48.136 elastic1066-production-search-eqiad
bswiki_archive_1717269969              0  r STARTED         84569    3.9mb 10.64.48.136 elastic1066-production-search-eqiad
commonswiki_file_1717000466            11 p STARTED       4442923   32.3gb 10.64.48.136 elastic1066-production-search-eqiad
commonswiki_file_1727809582            18 r STARTED       4862864   52.7gb 10.64.48.136 elastic1066-production-search-eqiad
commonswiki_general_1727903574         7  r STARTED       2056718   19.4gb 10.64.48.136 elastic1066-production-search-eqiad
commonswiki_titlesuggest_1709176056    0  r STARTED        203267   55.5mb 10.64.48.136 elastic1066-production-search-eqiad
cywiki_general_1728129378              0  p STARTED        169610  650.2mb 10.64.48.136 elastic1066-production-search-eqiad
dewiki_content_1727860262              7  r STARTED        573442   10.8gb 10.64.48.136 elastic1066-production-search-eqiad
enwiki_content_1728079212              1  r STARTED        628783   20.1gb 10.64.48.136 elastic1066-production-search-eqiad
enwiki_general_1727860262              16 r STARTED       1974243   27.2gb 10.64.48.136 elastic1066-production-search-eqiad
enwikisource_archive_1717279608        0  r STARTED         95665    6.5mb 10.64.48.136 elastic1066-production-search-eqiad
enwiki_titlesuggest_1740972983         2  r STARTED       2571305  931.8mb 10.64.48.136 elastic1066-production-search-eqiad
fiwiki_content_1728102665              0  r STARTED        223680    4.9gb 10.64.48.136 elastic1066-production-search-eqiad
frwiki_content_1727958460              1  r STARTED        464406   15.5gb 10.64.48.136 elastic1066-production-search-eqiad
frwiki_general_1727890968              0  r STARTED       1091279   12.1gb 10.64.48.136 elastic1066-production-search-eqiad
frwiktionary_general_1727932638        0  r STARTED        743390    3.8gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20230422                         0  r STARTED      38470320    2.1gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20231216                         0  r STARTED      42386027    2.6gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20240309                         1  r STARTED      43815148    2.4gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20240406                         0  r STARTED      44500744    2.5gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20240518                         1  r STARTED      45269679    2.7gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20240608                         0  r STARTED      45566957    2.5gb 10.64.48.136 elastic1066-production-search-eqiad
glent_20240629                         1  r STARTED      45858759      3gb 10.64.48.136 elastic1066-production-search-eqiad
hewikisource_archive_1717245858        0  r STARTED         63534    3.1mb 10.64.48.136 elastic1066-production-search-eqiad
hrwiki_archive_1717405558              0  r STARTED        186454    9.7mb 10.64.48.136 elastic1066-production-search-eqiad
huwiktionary_archive_1717328155        0  r STARTED          4648  361.8kb 10.64.48.136 elastic1066-production-search-eqiad
hywikisource_archive_1717316836        0  r STARTED          2403  259.1kb 10.64.48.136 elastic1066-production-search-eqiad
idwiki_titlesuggest_1740980543         0  p STARTED        854559  261.5mb 10.64.48.136 elastic1066-production-search-eqiad
iowiktionary_content_1728117860        0  p STARTED        329847  784.8mb 10.64.48.136 elastic1066-production-search-eqiad
iswiki_general_1727965990              0  p STARTED         57611  307.3mb 10.64.48.136 elastic1066-production-search-eqiad
itwikibooks_archive_1717346073         0  p STARTED          3830    422kb 10.64.48.136 elastic1066-production-search-eqiad
itwiki_content_1727938747              3  r STARTED        328209    8.2gb 10.64.48.136 elastic1066-production-search-eqiad
itwiki_general_1727868002              7  r STARTED        655697    6.2gb 10.64.48.136 elastic1066-production-search-eqiad
itwikiquote_general_1727935063         0  r STARTED        114893  437.4mb 10.64.48.136 elastic1066-production-search-eqiad
itwikiquote_titlesuggest_1724304032    0  p STARTED         64018   16.7mb 10.64.48.136 elastic1066-production-search-eqiad
itwikisource_content_1727874287        0  p STARTED        351448    4.4gb 10.64.48.136 elastic1066-production-search-eqiad
itwiki_titlesuggest_1740981166         1  r STARTED       1203870  307.8mb 10.64.48.136 elastic1066-production-search-eqiad
jawiki_content_1727985243              1  r STARTED        254582   11.6gb 10.64.48.136 elastic1066-production-search-eqiad
jawikisource_titlesuggest_1674708973   0  r STARTED         17840    8.3mb 10.64.48.136 elastic1066-production-search-eqiad
jvwiki_content_1728049976              0  r STARTED         90409 1001.1mb 10.64.48.136 elastic1066-production-search-eqiad
knwikisource_general_1727930176        0  p STARTED          6310     81mb 10.64.48.136 elastic1066-production-search-eqiad
kowikisource_content_1728064474        0  r STARTED         58089    4.2gb 10.64.48.136 elastic1066-production-search-eqiad
ltwiktionary_archive_1717247749        0  r STARTED         13917  656.1kb 10.64.48.136 elastic1066-production-search-eqiad
lvwiki_titlesuggest_1709271830         0  r STARTED        201466   54.6mb 10.64.48.136 elastic1066-production-search-eqiad
metawiki_archive_1717251387            0  p STARTED        292805     17mb 10.64.48.136 elastic1066-production-search-eqiad
metawiki_general_1727872329            2  r STARTED       1534778   10.8gb 10.64.48.136 elastic1066-production-search-eqiad
mgwiktionary_content_1728095592        3  p STARTED       1268887    2.4gb 10.64.48.136 elastic1066-production-search-eqiad
mrwiki_general_1728133488              0  r STARTED        134608    1.2gb 10.64.48.136 elastic1066-production-search-eqiad
ndswiki_archive_1717334088             0  r STARTED          1654    147kb 10.64.48.136 elastic1066-production-search-eqiad
newwiki_archive_1717201212             0  p STARTED         33368    1.2mb 10.64.48.136 elastic1066-production-search-eqiad
nowiki_content_1728107303              1  r STARTED        264834    3.5gb 10.64.48.136 elastic1066-production-search-eqiad
nowiki_titlesuggest_1740984421         0  r STARTED        809751  196.6mb 10.64.48.136 elastic1066-production-search-eqiad
ptwiki_content_1728003051              4  r STARTED        201176    5.1gb 10.64.48.136 elastic1066-production-search-eqiad
rowiki_content_1728106428              0  r STARTED        259695    4.6gb 10.64.48.136 elastic1066-production-search-eqiad
rowikisource_titlesuggest_1709188999   0  p STARTED         16395    7.9mb 10.64.48.136 elastic1066-production-search-eqiad
ruwiki_general_1727923233              6  r STARTED        413843    3.5gb 10.64.48.136 elastic1066-production-search-eqiad
ruwikinews_content_1728098112          1  p STARTED        374687     10gb 10.64.48.136 elastic1066-production-search-eqiad
ruwiki_titlesuggest_1740986451         0  r STARTED       1012384  441.4mb 10.64.48.136 elastic1066-production-search-eqiad
sawiktionary_content_1728068508        0  r STARTED        262225    2.3gb 10.64.48.136 elastic1066-production-search-eqiad
slwikisource_archive_1717308854        0  p STARTED          2257  235.4kb 10.64.48.136 elastic1066-production-search-eqiad
sqwiki_general_1728130624              0  r STARTED        152042      1gb 10.64.48.136 elastic1066-production-search-eqiad
svwiki_content_1727953317              6  r STARTED        588268    5.7gb 10.64.48.136 elastic1066-production-search-eqiad
thwiki_archive_1717218947              0  r STARTED        170664     12mb 10.64.48.136 elastic1066-production-search-eqiad
thwikibooks_archive_1717316543         0  p STARTED          6368  489.4kb 10.64.48.136 elastic1066-production-search-eqiad
thwiki_general_1727955003              0  r STARTED        724191    4.3gb 10.64.48.136 elastic1066-production-search-eqiad
ukwiki_content_1727946044              3  r STARTED        335742    5.1gb 10.64.48.136 elastic1066-production-search-eqiad
viwiki_general_1728073492              0  r STARTED       3557061   23.8gb 10.64.48.136 elastic1066-production-search-eqiad
vowiki_content_1728063857              0  r STARTED         40495  172.9mb 10.64.48.136 elastic1066-production-search-eqiad
wikidatawiki_content_1727882536        17 p STARTED       5963852   31.4gb 10.64.48.136 elastic1066-production-search-eqiad
wikidatawiki_content_1728009803        8  r STARTED       6076275     42gb 10.64.48.136 elastic1066-production-search-eqiad
wikidatawiki_general_1728093363        0  r STARTED        674518   25.2gb 10.64.48.136 elastic1066-production-search-eqiad

It has some hot shards like:

  • enwiki_content
  • enwiki_titlesuggest
  • dewiki_content
  • frwiki_content

Shard search stats for enwiki_content@1 is:
curl -s localhost:9200/_stats/search?level=shards | jq '.indices.enwiki_content_1728079212.shards."1"'

[
  {
    "routing": {
      "state": "STARTED",
      "primary": true,
      "node": "FovTizPSSpaKmBzxoCJpmQ",
      "relocating_node": null
    },
    "search": {
      "open_contexts": 3,
      "query_total": 26194447,
      "query_time_in_millis": 1335665831,
      "query_current": 1,
      "fetch_total": 7694799,
      "fetch_time_in_millis": 75924331,
      "fetch_current": 0,
      "scroll_total": 0,
      "scroll_time_in_millis": 0,
      "scroll_current": 0,
      "suggest_total": 0,
      "suggest_time_in_millis": 0,
      "suggest_current": 0
    },
    "commit": {
      "id": "CLZFsWF55Q2ox+0PwCP8UQ==",
      "generation": 903,
      "user_data": {
        "local_checkpoint": "9610978",
        "max_unsafe_auto_id_timestamp": "-1",
        "min_retained_seq_no": "9610103",
        "translog_uuid": "7zPoTrr0TaS3RctO6repvg",
        "history_uuid": "4WeVVHR8Tl2J1uMIrQ2HaQ",
        "max_seq_no": "9610978"
      },
      "num_docs": 628782
    },
    "seq_no": {
      "max_seq_no": 9613187,
      "local_checkpoint": 9613187,
      "global_checkpoint": 9613187
    },
    "retention_leases": {
      "primary_term": 3,
      "version": 1228576,
      "leases": [
        {
          "id": "peer_recovery/FovTizPSSpaKmBzxoCJpmQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        },
        {
          "id": "peer_recovery/ocZjIxEhQb-TojSu1v6NYQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        },
        {
          "id": "peer_recovery/eS9G_qj7RlC5dFdhH12MMQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        }
      ]
    },
    "shard_path": {
      "state_path": "/srv/elasticsearch/production-search-eqiad/nodes/0",
      "data_path": "/srv/elasticsearch/production-search-eqiad/nodes/0",
      "is_custom_data_path": false
    }
  },
  {
    "routing": {
      "state": "STARTED",
      "primary": false,
      "node": "ocZjIxEhQb-TojSu1v6NYQ",
      "relocating_node": null
    },
    "search": {
      "open_contexts": 3,
      "query_total": 29366891,
      "query_time_in_millis": 1415151460,
      "query_current": 2,
      "fetch_total": 8706023,
      "fetch_time_in_millis": 83860472,
      "fetch_current": 0,
      "scroll_total": 0,
      "scroll_time_in_millis": 0,
      "scroll_current": 0,
      "suggest_total": 0,
      "suggest_time_in_millis": 0,
      "suggest_current": 0
    },
    "commit": {
      "id": "LOmO9FFutdoPNzAIqdNGbw==",
      "generation": 907,
      "user_data": {
        "local_checkpoint": "9600049",
        "max_unsafe_auto_id_timestamp": "-1",
        "min_retained_seq_no": "9597758",
        "translog_uuid": "a44BURyfSVm5IuYKpDZTJQ",
        "history_uuid": "4WeVVHR8Tl2J1uMIrQ2HaQ",
        "max_seq_no": "9600049"
      },
      "num_docs": 628777
    },
    "seq_no": {
      "max_seq_no": 9613187,
      "local_checkpoint": 9613187,
      "global_checkpoint": 9613187
    },
    "retention_leases": {
      "primary_term": 3,
      "version": 1228576,
      "leases": [
        {
          "id": "peer_recovery/FovTizPSSpaKmBzxoCJpmQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        },
        {
          "id": "peer_recovery/ocZjIxEhQb-TojSu1v6NYQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        },
        {
          "id": "peer_recovery/eS9G_qj7RlC5dFdhH12MMQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        }
      ]
    },
    "shard_path": {
      "state_path": "/srv/elasticsearch/production-search-eqiad/nodes/0",
      "data_path": "/srv/elasticsearch/production-search-eqiad/nodes/0",
      "is_custom_data_path": false
    }
  },
  {
    "routing": {
      "state": "STARTED",
      "primary": false,
      "node": "eS9G_qj7RlC5dFdhH12MMQ",
      "relocating_node": null
    },
    "search": {
      "open_contexts": 4,
      "query_total": 29269774,
      "query_time_in_millis": 1885608565,
      "query_current": 1,
      "fetch_total": 8834836,
      "fetch_time_in_millis": 106161584,
      "fetch_current": 0,
      "scroll_total": 0,
      "scroll_time_in_millis": 0,
      "scroll_current": 0,
      "suggest_total": 0,
      "suggest_time_in_millis": 0,
      "suggest_current": 0
    },
    "commit": {
      "id": "xUeygehX0MfXHMfrGyCJ/g==",
      "generation": 910,
      "user_data": {
        "local_checkpoint": "9611743",
        "max_unsafe_auto_id_timestamp": "-1",
        "min_retained_seq_no": "9592030",
        "translog_uuid": "aajb9KvgQ_yNPKiJKlVztw",
        "history_uuid": "4WeVVHR8Tl2J1uMIrQ2HaQ",
        "max_seq_no": "9611749"
      },
      "num_docs": 628783
    },
    "seq_no": {
      "max_seq_no": 9613187,
      "local_checkpoint": 9613187,
      "global_checkpoint": 9613187
    },
    "retention_leases": {
      "primary_term": 3,
      "version": 1228576,
      "leases": [
        {
          "id": "peer_recovery/FovTizPSSpaKmBzxoCJpmQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        },
        {
          "id": "peer_recovery/ocZjIxEhQb-TojSu1v6NYQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        },
        {
          "id": "peer_recovery/eS9G_qj7RlC5dFdhH12MMQ",
          "retaining_seq_no": 9613186,
          "timestamp": 1740993196111,
          "source": "peer recovery"
        }
      ]
    },
    "shard_path": {
      "state_path": "/srv/elasticsearch/production-search-eqiad/nodes/0",
      "data_path": "/srv/elasticsearch/production-search-eqiad/nodes/0",
      "is_custom_data_path": false
    }
  }
]