I believe this is related to T383814: Upgrade eventgate-wikimedia to node20 since the timing pretty much lines up. Current NEL logs (schema /w3c/reportingapi/network_error/1.0.0) in logstash for index w3creportingapi don't have geoip information anymore. NEL dashboard at the time of breakage: https://logstash.wikimedia.org/goto/8b11cf7faba31f31fe703449df7040f2
{
"_index": "w3creportingapi-default-1-1.0.0-3-2025.10",
"_id": "itwdYZUB-jfKnJiswTLo",
"_version": 1,
"_score": null,
"_source": {
"uri_host": "measure-esams.wikimedia.org",
"tags": [
"input-kafka-networkerror-eqiad",
"kafka",
"throttle-exempt"
],
"type": "network-error",
"body": {
"protocol": "h2",
"status_code": 200,
"server_ip": "185.15.59.240",
"sampling_fraction": 1,
"elapsed_time": 160,
"method": "GET",
"type": "ok",
"phase": "application",
"referrer": "https://en.wikipedia.org/"
},
"@timestamp": "2025-03-04T12:24:54.795Z",
"age": 15164,
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0",
"$schema": "/w3c/reportingapi/network_error/1.0.0",
"meta": {
"dt": "2025-03-04T12:24:54.741Z",
"stream": "w3c.reportingapi.network_error",
"request_id": "433efd09-2d2b-4fdc-8671-0d9ba67a4c57",
"id": "c9f4cf8f-1ead-4a1b-b738-380382e8f40c"
},
"http": {
"request_headers": {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0"
},
"client_ip": "a.b.c.d"
},
"url": "https://measure-esams.wikimedia.org/measure?pulse_identifier=f1a2f5b25c6040b0097c_esams_0&pulse_number=0"
},
"fields": {
"isp_asn": [
"unknown (unknown)"
],
"meta.dt": [
"2025-03-04T12:24:54.741Z"
],
"@timestamp": [
"2025-03-04T12:24:54.795Z"
],
"uri_host": [
"measure-esams.wikimedia.org"
]
},
"sort": [
1741091094795
]
}compare it with an older entry
{
"_index": "w3creportingapi-default-1-1.0.0-3-2025.09",
"_id": "npFpV5UBaJATmKuI29-h",
"_version": 1,
"_score": null,
"_source": {
"tags": [
"input-kafka-networkerror-codfw",
"kafka",
"throttle-exempt"
],
"@timestamp": "2025-03-02T15:11:49.755Z",
"type": "network-error",
"$schema": "/w3c/reportingapi/network_error/1.0.0",
"age": 13824,
"uri_host": "measure-eqsin.wikimedia.org",
"http": {
"client_ip": "a.b.c.d",
"request_headers": {
"x-geoip-isp": "Comcast Cable",
"x-geoip-country": "US",
"x-geoip-organization": "Comcast Cable",
"x-geoip-as-number": "7922",
"x-geoip-subdivision": "FL",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0"
}
},
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0",
"meta": {
"request_id": "685c9169-91b7-4bd7-95f4-a2b2ebefc578",
"stream": "w3c.reportingapi.network_error",
"id": "9d2f4fe7-9733-4470-8b68-031e208c0f60",
"dt": "2025-03-02T15:11:49.680Z"
},
"body": {
"protocol": "h2",
"method": "GET",
"server_ip": "2001:df2:e500:ed1a::2:b",
"referrer": "https://en.wikipedia.org/",
"status_code": 200,
"sampling_fraction": 1,
"elapsed_time": 259,
"type": "ok",
"phase": "application"
},
"url": "https://measure-eqsin.wikimedia.org/measure?pulse_identifier=63c6a729a01bc9eadb2c_eqsin_1&pulse_number=1"
},
"fields": {
"isp_asn": [
"Comcast Cable (7922)"
],
"meta.dt": [
"2025-03-02T15:11:49.680Z"
],
"@timestamp": [
"2025-03-02T15:11:49.755Z"
],
"uri_host": [
"measure-eqsin.wikimedia.org"
]
},
"sort": [
1740928309755
]
}The metric disappears at the same time as the deploy:
https://sal.toolforge.org/log/svLoXJUBvg159pQrWDEp 16:48 <otto@deploy2002> helmfile [codfw] DONE helmfile.d/services/eventgate-logging-external: apply [production]