Page MenuHomePhabricator
Paste P12807

enwiki_general_1587198756 allocation failure elasticsearch chi@eqiad
ActivePublic
Actions

Authored by dcausse on Sep 28 2020, 7:01 AM.
{
"index": "enwiki_general_1587198756",
"shard": 0,
"primary": false,
"current_state": "unassigned",
"unassigned_info": {
"reason": "ALLOCATION_FAILED",
"at": "2020-09-24T14:34:37.564Z",
"failed_allocation_attempts": 5,
"details": "failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ",
"last_allocation_status": "no_attempt"
},
"can_allocate": "no",
"allocate_explanation": "cannot allocate because allocation is not permitted to any of the nodes",
"node_allocation_decisions": [
{
"node_id": "-i-P7rO-R-mtxeOsOnXRng",
"node_name": "elastic1054-production-search-eqiad",
"transport_address": "10.64.0.115:9300",
"node_attributes": {
"hostname": "elastic1054",
"rack": "A3",
"row": "A",
"fqdn": "elastic1054.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "2gqqvxcITQ2Anngr7K7zSQ",
"node_name": "elastic1033-production-search-eqiad",
"transport_address": "10.64.0.234:9300",
"node_attributes": {
"hostname": "elastic1033",
"rack": "A3",
"row": "A",
"fqdn": "elastic1033.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "3otTviKAQ6K80-xOytI97Q",
"node_name": "elastic1047-production-search-eqiad",
"transport_address": "10.64.16.71:9300",
"node_attributes": {
"hostname": "elastic1047",
"rack": "B6",
"row": "B",
"fqdn": "elastic1047.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "65SkRqtuQ22ydt-GqFgDIg",
"node_name": "elastic1044-production-search-eqiad",
"transport_address": "10.64.0.85:9300",
"node_attributes": {
"hostname": "elastic1044",
"rack": "A6",
"row": "A",
"fqdn": "elastic1044.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "6XgWbRLLTCS0xhK6DYV3mA",
"node_name": "elastic1036-production-search-eqiad",
"transport_address": "10.64.16.45:9300",
"node_attributes": {
"hostname": "elastic1036",
"rack": "B3",
"row": "B",
"fqdn": "elastic1036.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "7PwdfgT-R7auzYHTDLHXBw",
"node_name": "elastic1059-production-search-eqiad",
"transport_address": "10.64.32.95:9300",
"node_attributes": {
"hostname": "elastic1059",
"rack": "C8",
"row": "C",
"fqdn": "elastic1059.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "9TSoz0HcSq6IerXtFpEqaA",
"node_name": "elastic1066-production-search-eqiad",
"transport_address": "10.64.48.136:9300",
"node_attributes": {
"hostname": "elastic1066",
"rack": "D6",
"row": "D",
"fqdn": "elastic1066.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "AlB38EnKSkKdnZkIOUA3Hg",
"node_name": "elastic1042-production-search-eqiad",
"transport_address": "10.64.32.110:9300",
"node_attributes": {
"hostname": "elastic1042",
"rack": "C5",
"row": "C",
"fqdn": "elastic1042.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "BeVunwq_THitzNmRJdeRWg",
"node_name": "elastic1065-production-search-eqiad",
"transport_address": "10.64.48.135:9300",
"node_attributes": {
"hostname": "elastic1065",
"rack": "D5",
"row": "D",
"fqdn": "elastic1065.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "D_gxN3s2SaCxQq8-4KiJGg",
"node_name": "elastic1048-production-search-eqiad",
"transport_address": "10.64.0.238:9300",
"node_attributes": {
"hostname": "elastic1048",
"rack": "A6",
"row": "A",
"fqdn": "elastic1048.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "IKJG2DpZT1uKCzXLYIjzAw",
"node_name": "elastic1050-production-search-eqiad",
"transport_address": "10.64.16.112:9300",
"node_attributes": {
"hostname": "elastic1050",
"rack": "B4",
"row": "B",
"fqdn": "elastic1050.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "Lo3rq0e-Q7eRnjVOGUmW5w",
"node_name": "elastic1060-production-search-eqiad",
"transport_address": "10.64.48.130:9300",
"node_attributes": {
"hostname": "elastic1060",
"rack": "D1",
"row": "D",
"fqdn": "elastic1060.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "M8DJGlbIQ4yXvSTDcwZoBw",
"node_name": "elastic1051-production-search-eqiad",
"transport_address": "10.64.32.21:9300",
"node_attributes": {
"hostname": "elastic1051",
"rack": "C7",
"row": "C",
"fqdn": "elastic1051.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "Mk_LhazQQb2Qd_ulCqOAuQ",
"node_name": "elastic1057-production-search-eqiad",
"transport_address": "10.64.32.93:9300",
"node_attributes": {
"hostname": "elastic1057",
"rack": "C3",
"row": "C",
"fqdn": "elastic1057.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "same_shard",
"decision": "NO",
"explanation": "the shard cannot be allocated to the same node on which a copy of the shard already exists [[enwiki_general_1587198756][0], node[Mk_LhazQQb2Qd_ulCqOAuQ], [P], s[STARTED], a[id=JEuRvJQvTAuui6gIpkaWJQ]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "NAAhKTkFSYei-UCLvtxJKg",
"node_name": "elastic1049-production-search-eqiad",
"transport_address": "10.64.16.111:9300",
"node_attributes": {
"hostname": "elastic1049",
"rack": "B4",
"row": "B",
"fqdn": "elastic1049.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "Opco8YWwT5eQlQzhgrxozQ",
"node_name": "elastic1037-production-search-eqiad",
"transport_address": "10.64.16.46:9300",
"node_attributes": {
"hostname": "elastic1037",
"rack": "B3",
"row": "B",
"fqdn": "elastic1037.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "OzF9aPpmQy6kwNEdq1irrw",
"node_name": "elastic1063-production-search-eqiad",
"transport_address": "10.64.48.133:9300",
"node_attributes": {
"hostname": "elastic1063",
"rack": "D3",
"row": "D",
"fqdn": "elastic1063.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "shards_limit",
"decision": "NO",
"explanation": "too many shards [3] allocated to this node for index [enwiki_general_1587198756], index setting [index.routing.allocation.total_shards_per_node=3]"
}
]
},
{
"node_id": "QSrKafz0Qb-iwYWmGbw47w",
"node_name": "elastic1032-production-search-eqiad",
"transport_address": "10.64.0.233:9300",
"node_attributes": {
"hostname": "elastic1032",
"rack": "A3",
"row": "A",
"fqdn": "elastic1032.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "TcK3ER2cTPaRo-Ty4545gg",
"node_name": "elastic1056-production-search-eqiad",
"transport_address": "10.64.16.132:9300",
"node_attributes": {
"hostname": "elastic1056",
"rack": "B5",
"row": "B",
"fqdn": "elastic1056.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "WgdcUhYUQzGW2EpKsWwTlw",
"node_name": "elastic1053-production-search-eqiad",
"transport_address": "10.64.0.114:9300",
"node_attributes": {
"hostname": "elastic1053",
"rack": "A5",
"row": "A",
"fqdn": "elastic1053.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "XEGBDBEbRoCuGAbvaWwjTg",
"node_name": "elastic1062-production-search-eqiad",
"transport_address": "10.64.48.132:9300",
"node_attributes": {
"hostname": "elastic1062",
"rack": "D3",
"row": "D",
"fqdn": "elastic1062.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "Z0vuGmzxS1mXGERbKmaXmg",
"node_name": "elastic1061-production-search-eqiad",
"transport_address": "10.64.48.131:9300",
"node_attributes": {
"hostname": "elastic1061",
"rack": "D1",
"row": "D",
"fqdn": "elastic1061.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "boGnhtCUTWSXEYD2JX_zBQ",
"node_name": "elastic1064-production-search-eqiad",
"transport_address": "10.64.48.134:9300",
"node_attributes": {
"hostname": "elastic1064",
"rack": "D4",
"row": "D",
"fqdn": "elastic1064.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "cBRsRG7hRk6E2tqPb4agSQ",
"node_name": "elastic1041-production-search-eqiad",
"transport_address": "10.64.32.109:9300",
"node_attributes": {
"hostname": "elastic1041",
"rack": "C5",
"row": "C",
"fqdn": "elastic1041.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "dU3Z0hOtSEqrRkiS707UKA",
"node_name": "elastic1052-production-search-eqiad",
"transport_address": "10.64.32.22:9300",
"node_attributes": {
"hostname": "elastic1052",
"rack": "C7",
"row": "C",
"fqdn": "elastic1052.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "dqJZXXNpSx-QUOCmbfpdVA",
"node_name": "elastic1055-production-search-eqiad",
"transport_address": "10.64.16.131:9300",
"node_attributes": {
"hostname": "elastic1055",
"rack": "B1",
"row": "B",
"fqdn": "elastic1055.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "e4WY1V5PQoa4lnzwxmM1nQ",
"node_name": "elastic1038-production-search-eqiad",
"transport_address": "10.64.16.47:9300",
"node_attributes": {
"hostname": "elastic1038",
"rack": "B3",
"row": "B",
"fqdn": "elastic1038.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "elMmwSheQXWnFZSdArRFlA",
"node_name": "elastic1034-production-search-eqiad",
"transport_address": "10.64.0.235:9300",
"node_attributes": {
"hostname": "elastic1034",
"rack": "A3",
"row": "A",
"fqdn": "elastic1034.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "hCP8oBWATB-ow4_46CiODw",
"node_name": "elastic1043-production-search-eqiad",
"transport_address": "10.64.32.111:9300",
"node_attributes": {
"hostname": "elastic1043",
"rack": "C5",
"row": "C",
"fqdn": "elastic1043.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "joyZCBJdS7eU3ijs0kmNww",
"node_name": "elastic1046-production-search-eqiad",
"transport_address": "10.64.16.70:9300",
"node_attributes": {
"hostname": "elastic1046",
"rack": "B6",
"row": "B",
"fqdn": "elastic1046.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "same_shard",
"decision": "NO",
"explanation": "the shard cannot be allocated to the same node on which a copy of the shard already exists [[enwiki_general_1587198756][0], node[joyZCBJdS7eU3ijs0kmNww], [R], s[STARTED], a[id=u0GAPbkDQDSyBYtDIpn5Pg]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "kmWVLVQfS8SL9cTlA-0k0Q",
"node_name": "elastic1058-production-search-eqiad",
"transport_address": "10.64.32.94:9300",
"node_attributes": {
"hostname": "elastic1058",
"rack": "C3",
"row": "C",
"fqdn": "elastic1058.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "nMtaY80VRamXRC3_sxDsXA",
"node_name": "elastic1045-production-search-eqiad",
"transport_address": "10.64.0.86:9300",
"node_attributes": {
"hostname": "elastic1045",
"rack": "A6",
"row": "A",
"fqdn": "elastic1045.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "ohzZxJhqSoSEE_rkbGiVkw",
"node_name": "elastic1040-production-search-eqiad",
"transport_address": "10.64.32.108:9300",
"node_attributes": {
"hostname": "elastic1040",
"rack": "C5",
"row": "C",
"fqdn": "elastic1040.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
},
{
"node_id": "pCFROq6mRQqF0ir0DG3mfQ",
"node_name": "elastic1035-production-search-eqiad",
"transport_address": "10.64.0.236:9300",
"node_attributes": {
"hostname": "elastic1035",
"rack": "A3",
"row": "A",
"fqdn": "elastic1035.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "r1YT_cXoScCSxnFP48iYZA",
"node_name": "elastic1067-production-search-eqiad",
"transport_address": "10.64.48.137:9300",
"node_attributes": {
"hostname": "elastic1067",
"rack": "D6",
"row": "D",
"fqdn": "elastic1067.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
}
]
},
{
"node_id": "zE6Jm7ZaRFCS3IoFPwJtmw",
"node_name": "elastic1039-production-search-eqiad",
"transport_address": "10.64.16.48:9300",
"node_attributes": {
"hostname": "elastic1039",
"rack": "B3",
"row": "B",
"fqdn": "elastic1039.eqiad.wmnet"
},
"node_decision": "no",
"deciders": [
{
"decider": "max_retry",
"decision": "NO",
"explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-09-24T14:34:37.564Z], failed_attempts[5], delayed=false, details[failed shard on node [Lo3rq0e-Q7eRnjVOGUmW5w]: failed to create shard, failure IOException[failed to obtain in-memory shard lock]; nested: ShardLockObtainFailedException[[enwiki_general_1587198756][0]: obtaining shard lock timed out after 5000ms]; ], allocation_status[no_attempt]]]"
},
{
"decider": "awareness",
"decision": "NO",
"explanation": "there are too many copies of the shard allocated to nodes with attribute [row], there are [3] total configured shard copies for this shard id and [4] total attribute values, expected the allocated shard count per attribute [2] to be less than or equal to the upper bound of the required number of shards per attribute [1]"
}
]
}
]
}
Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL