I noticed this while running `wmcs.openstack.cloudvirt.safe_reboot`, taking for example cloudvirt1049 it looks like the silence is issued, and yet `NeutronAgentDown` fires (from irc)
```
2026-07-06T12:17:50 -jinxer-wm:#wikimedia-cloud-feed- FIRING: NeutronAgentDown: Neutron neutron-openvswitch-agent on cloudvirt1049 is down - https://wikitech.wikimedia.org/wiki/Portal:Cloud_VPS/Admin/Troubleshooting#Networking_failures - https://grafana.wikimedia.org/d/wKnDJf97z/wmcs-neutron-eqiad1 - https://alerts.wikimedia.org/?q=alertname%3DNeutronAgentDown
2026-07-06T12:22:49 -jinxer-wm:#wikimedia-cloud-feed- RESOLVED: NeutronAgentDown: Neutron neutron-openvswitch-agent on cloudvirt1049 is down - https://wikitech.wikimedia.org/wiki/Portal:Cloud_VPS/Admin/Troubleshooting#Networking_failures - https://grafana.wikimedia.org/d/wKnDJf97z/wmcs-neutron-eqiad1 - https://alerts.wikimedia.org/?q=alertname%3DNeutronAgentDown
```
This is what the cookbook did in terms of silence:
```
2026-07-06 11:07:44,946 filippo 1908085 [INFO] Created silence ID 11afc5da-55fd-4d1c-ad8a-8f67bb0d9925
2026-07-06 12:18:49,022 filippo 1908085 [INFO] Deleted silence ID 11afc5da-55fd-4d1c-ad8a-8f67bb0d9925
```
And the silence ID looks like this
{F92097831}
In other words it looks like `NeutronAgentDown` should not have fired at `12:17:50` because the silence was still in place and was deleted at about one minute later at `12:18:49`
The alert json logged in logstash looks like this (alert firing)
```lines=10
{
"_index": "ecs-alerts-2-1.11.0-8-2026",
"_id": "g_ddN58By4EOfO-FAnpT",
"_version": 1,
"_score": null,
"_source": {
"log": {
"level": "INFO",
"syslog": {
"priority": 30,
"severity": {
"code": 6,
"name": "informational"
},
"facility": {
"code": 3,
"name": "daemon"
}
}
},
"ecs": {
"version": "1.7.0"
},
"labels": {
"status": "firing",
"team": "wmcs",
"source": "prometheus",
"adminState": "up",
"hostname": "cloudvirt1049",
"site": "eqiad",
"generatorURL": "https://prometheus-eqiad.wikimedia.org/cloud/graph?g0.expr=openstack_neutron_agent_state%7BadminState%3D%22up%22%7D+%3D%3D+0&g0.tab=1",
"instance": "cloudvirt1049",
"alertname": "NeutronAgentDown",
"cluster": "wmcs",
"deployment": "eqiad1",
"prometheus": "cloud",
"service": "neutron-openvswitch-agent",
"id": "752c7386-abf2-491b-9fef-7e8d934e4060",
"alert_severity": "warning",
"summary": "Neutron neutron-openvswitch-agent on cloudvirt1049 is down",
"job": "openstack"
},
"tags": [
"rsyslog-shipper",
"kafka",
"es",
"normalized_message_untrimmed"
],
"service": {
"type": "alertmanager-webhook-logger"
},
"event": {
"original": "{\"externalURL\":\"http://alert2002:9093\",\"dashboard\":\"https://grafana.wikimedia.org/d/wKnDJf97z/wmcs-neutron-eqiad1\",\"job\":\"openstack\",\"receiver\":\"logger\",\"source\":\"prometheus\",\"adminState\":\"up\",\"hostname\":\"cloudvirt1049\",\"rsyslog.facility\":\"daemon\",\"runbook\":\"https://wikitech.wikimedia.org/wiki/Portal:Cloud_VPS/Admin/Troubleshooting#Networking_failures\",\"site\":\"eqiad\",\"alertname\":\"NeutronAgentDown\",\"cluster\":\"wmcs\",\"timestamp\":\"2026-07-06T12:17:49.789003624Z\",\"msg\":\"<30>Jul 6 12:17:49 alertmanager-webhook-logger[825]:{\\\"adminState\\\":\\\"up\\\",\\\"alertname\\\":\\\"NeutronAgentDown\\\",\\\"cluster\\\":\\\"wmcs\\\",\\\"dashboard\\\":\\\"https://grafana.wikimedia.org/d/wKnDJf97z/wmcs-neutron-eqiad1\\\",\\\"deployment\\\":\\\"eqiad1\\\",\\\"description\\\":\\\"A Neutron agent is down, VMs will have connectivity issues\\\",\\\"endsAt\\\":\\\"0001-01-01T00:00:00Z\\\",\\\"externalURL\\\":\\\"http://alert2002:9093\\\",\\\"generatorURL\\\":\\\"https://prometheus-eqiad.wikimedia.org/cloud/graph?g0.expr=openstack_neutron_agent_state%7BadminState%3D%22up%22%7D+%3D%3D+0\\\\u0026g0.tab=1\\\",\\\"hostname\\\":\\\"cloudvirt1049\\\",\\\"id\\\":\\\"752c7386-abf2-491b-9fef-7e8d934e4060\\\",\\\"instance\\\":\\\"cloudvirt1049\\\",\\\"job\\\":\\\"openstack\\\",\\\"prometheus\\\":\\\"cloud\\\",\\\"receiver\\\":\\\"logger\\\",\\\"runbook\\\":\\\"https://wikitech.wikimedia.org/wiki/Portal:Cloud_VPS/Admin/Troubleshooting#Networking_failures\\\",\\\"service\\\":\\\"neutron-openvswitch-agent\\\",\\\"severity\\\":\\\"warning\\\",\\\"site\\\":\\\"eqiad\\\",\\\"source\\\":\\\"prometheus\\\",\\\"startsAt\\\":\\\"2026-07-06T12:17:19.765Z\\\",\\\"status\\\":\\\"firing\\\",\\\"summary\\\":\\\"Neutron neutron-openvswitch-agent on cloudvirt1049 is down\\\",\\\"team\\\":\\\"wmcs\\\",\\\"timestamp\\\":\\\"2026-07-06T12:17:49.789003624Z\\\"}\",\"service\":\"neutron-openvswitch-agent\",\"tags\":[\"rsyslog-shipper\",\"kafka\",\"es\"],\"id\":\"752c7386-abf2-491b-9fef-7e8d934e4060\",\"description\":\"A Neutron agent is down, VMs will have connectivity issues\",\"summary\":\"Neutron neutron-openvswitch-agent on cloudvirt1049 is down\",\"status\":\"firing\",\"type\":\"syslog\",\"team\":\"wmcs\",\"startsAt\":\"2026-07-06T12:17:19.765Z\",\"rsyslog.programname\":\"alertmanager-webhook-logger\",\"rsyslog.timereported\":\"2026-07-06T12:17:49.789850+00:00\",\"@version\":\"1\",\"program\":\"alertmanager-webhook-logger\",\"severity\":\"warning\",\"rsyslog.severity\":\"info\",\"generatorURL\":\"https://prometheus-eqiad.wikimedia.org/cloud/graph?g0.expr=openstack_neutron_agent_state%7BadminState%3D%22up%22%7D+%3D%3D+0&g0.tab=1\",\"instance\":\"cloudvirt1049\",\"logsource\":\"alert2002\",\"deployment\":\"eqiad1\",\"prometheus\":\"cloud\",\"@timestamp\":\"2026-07-06T12:17:49.900Z\",\"host\":\"alert2002\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"facility\":\"daemon\",\"rsyslog.hostname\":\"alert2002\"}",
"kind": "alert"
},
"message": "[firing] NeutronAgentDown (Neutron neutron-openvswitch-agent on cloudvirt1049 is down - A Neutron agent is down, VMs will have connectivity issues)",
"normalized": {
"message": "[firing] neutronagentdown (neutron neutron-openvswitch-agent on cloudvirt1049 is down - a neutron agent is down, vms will have connectivity issues)",
"dropped": {
"no_such_field": [
"[receiver]",
"[dashboard]",
"[normalized_message]",
"[runbook]",
"[rsyslog][programname]",
"[rsyslog][timereported]",
"[rsyslog][severity]",
"[rsyslog][facility]",
"[rsyslog][hostname]",
"[timestamp]",
"[description]",
"[startsAt]",
"[@version]",
"[logsource]",
"[endsAt]",
"[_log]",
"[externalURL]"
]
}
},
"@timestamp": "2026-07-06T12:17:49.789Z",
"host": {
"name": "alert2002"
}
},
"fields": {
"@timestamp": [
"2026-07-06T12:17:49.789Z"
]
},
"highlight": {
"service.type": [
"@opensearch-dashboards-highlighted-field@alertmanager-webhook-logger@/opensearch-dashboards-highlighted-field@"
],
"labels.instance": [
"@opensearch-dashboards-highlighted-field@cloudvirt1049@/opensearch-dashboards-highlighted-field@"
],
"labels.alertname": [
"@opensearch-dashboards-highlighted-field@NeutronAgentDown@/opensearch-dashboards-highlighted-field@"
]
},
"sort": [
1783340269789
]
}
```
and alert resolved
```lines=10
{
"_index": "ecs-alerts-2-1.11.0-8-2026",
"_id": "0f1hN58By4EOfO-FlXVK",
"_version": 1,
"_score": null,
"_source": {
"service": {
"type": "alertmanager-webhook-logger"
},
"ecs": {
"version": "1.7.0"
},
"event": {
"kind": "alert",
"original": "{\"deployment\":\"eqiad1\",\"service\":\"neutron-openvswitch-agent\",\"site\":\"eqiad\",\"source\":\"prometheus\",\"hostname\":\"cloudvirt1049\",\"rsyslog.programname\":\"alertmanager-webhook-logger\",\"msg\":\"<30>Jul 6 12:22:49 alertmanager-webhook-logger[825]:{\\\"adminState\\\":\\\"up\\\",\\\"alertname\\\":\\\"NeutronAgentDown\\\",\\\"cluster\\\":\\\"wmcs\\\",\\\"dashboard\\\":\\\"https://grafana.wikimedia.org/d/wKnDJf97z/wmcs-neutron-eqiad1\\\",\\\"deployment\\\":\\\"eqiad1\\\",\\\"description\\\":\\\"A Neutron agent is down, VMs will have connectivity issues\\\",\\\"endsAt\\\":\\\"2026-07-06T12:20:19.765Z\\\",\\\"externalURL\\\":\\\"http://alert2002:9093\\\",\\\"generatorURL\\\":\\\"https://prometheus-eqiad.wikimedia.org/cloud/graph?g0.expr=openstack_neutron_agent_state%7BadminState%3D%22up%22%7D+%3D%3D+0\\\\u0026g0.tab=1\\\",\\\"hostname\\\":\\\"cloudvirt1049\\\",\\\"id\\\":\\\"752c7386-abf2-491b-9fef-7e8d934e4060\\\",\\\"instance\\\":\\\"cloudvirt1049\\\",\\\"job\\\":\\\"openstack\\\",\\\"prometheus\\\":\\\"cloud\\\",\\\"receiver\\\":\\\"logger\\\",\\\"runbook\\\":\\\"https://wikitech.wikimedia.org/wiki/Portal:Cloud_VPS/Admin/Troubleshooting#Networking_failures\\\",\\\"service\\\":\\\"neutron-openvswitch-agent\\\",\\\"severity\\\":\\\"warning\\\",\\\"site\\\":\\\"eqiad\\\",\\\"source\\\":\\\"prometheus\\\",\\\"startsAt\\\":\\\"2026-07-06T12:17:19.765Z\\\",\\\"status\\\":\\\"resolved\\\",\\\"summary\\\":\\\"Neutron neutron-openvswitch-agent on cloudvirt1049 is down\\\",\\\"team\\\":\\\"wmcs\\\",\\\"timestamp\\\":\\\"2026-07-06T12:22:49.790474179Z\\\"}\",\"rsyslog.hostname\":\"alert2002\",\"description\":\"A Neutron agent is down, VMs will have connectivity issues\",\"receiver\":\"logger\",\"summary\":\"Neutron neutron-openvswitch-agent on cloudvirt1049 is down\",\"timestamp\":\"2026-07-06T12:22:49.790474179Z\",\"startsAt\":\"2026-07-06T12:17:19.765Z\",\"externalURL\":\"http://alert2002:9093\",\"id\":\"752c7386-abf2-491b-9fef-7e8d934e4060\",\"dashboard\":\"https://grafana.wikimedia.org/d/wKnDJf97z/wmcs-neutron-eqiad1\",\"generatorURL\":\"https://prometheus-eqiad.wikimedia.org/cloud/graph?g0.expr=openstack_neutron_agent_state%7BadminState%3D%22up%22%7D+%3D%3D+0&g0.tab=1\",\"status\":\"resolved\",\"facility\":\"daemon\",\"alertname\":\"NeutronAgentDown\",\"program\":\"alertmanager-webhook-logger\",\"rsyslog.facility\":\"daemon\",\"cluster\":\"wmcs\",\"prometheus\":\"cloud\",\"@version\":\"1\",\"instance\":\"cloudvirt1049\",\"logsource\":\"alert2002\",\"severity\":\"warning\",\"job\":\"openstack\",\"team\":\"wmcs\",\"type\":\"syslog\",\"@timestamp\":\"2026-07-06T12:22:49.901Z\",\"tags\":[\"rsyslog-shipper\",\"kafka\",\"es\"],\"rsyslog.severity\":\"info\",\"adminState\":\"up\",\"host\":\"alert2002\",\"endsAt\":\"2026-07-06T12:20:19.765Z\",\"runbook\":\"https://wikitech.wikimedia.org/wiki/Portal:Cloud_VPS/Admin/Troubleshooting#Networking_failures\",\"rsyslog.timereported\":\"2026-07-06T12:22:49.791071+00:00\"}"
},
"message": "[resolved] NeutronAgentDown (Neutron neutron-openvswitch-agent on cloudvirt1049 is down - A Neutron agent is down, VMs will have connectivity issues)",
"labels": {
"generatorURL": "https://prometheus-eqiad.wikimedia.org/cloud/graph?g0.expr=openstack_neutron_agent_state%7BadminState%3D%22up%22%7D+%3D%3D+0&g0.tab=1",
"service": "neutron-openvswitch-agent",
"site": "eqiad",
"status": "resolved",
"source": "prometheus",
"alertname": "NeutronAgentDown",
"hostname": "cloudvirt1049",
"cluster": "wmcs",
"prometheus": "cloud",
"alert_severity": "warning",
"instance": "cloudvirt1049",
"summary": "Neutron neutron-openvswitch-agent on cloudvirt1049 is down",
"job": "openstack",
"team": "wmcs",
"adminState": "up",
"id": "752c7386-abf2-491b-9fef-7e8d934e4060",
"deployment": "eqiad1"
},
"normalized": {
"message": "[resolved] neutronagentdown (neutron neutron-openvswitch-agent on cloudvirt1049 is down - a neutron agent is down, vms will have connectivity issues)",
"dropped": {
"no_such_field": [
"[normalized_message]",
"[description]",
"[receiver]",
"[timestamp]",
"[startsAt]",
"[_log]",
"[externalURL]",
"[dashboard]",
"[@version]",
"[logsource]",
"[rsyslog][severity]",
"[rsyslog][facility]",
"[rsyslog][timereported]",
"[rsyslog][programname]",
"[rsyslog][hostname]",
"[endsAt]",
"[runbook]"
]
}
},
"tags": [
"rsyslog-shipper",
"kafka",
"es",
"normalized_message_untrimmed"
],
"@timestamp": "2026-07-06T12:22:49.790Z",
"host": {
"name": "alert2002"
},
"log": {
"level": "INFO",
"syslog": {
"severity": {
"code": 6,
"name": "informational"
},
"facility": {
"code": 3,
"name": "daemon"
},
"priority": 30
}
}
},
"fields": {
"@timestamp": [
"2026-07-06T12:22:49.790Z"
]
},
"highlight": {
"service.type": [
"@opensearch-dashboards-highlighted-field@alertmanager-webhook-logger@/opensearch-dashboards-highlighted-field@"
],
"labels.instance": [
"@opensearch-dashboards-highlighted-field@cloudvirt1049@/opensearch-dashboards-highlighted-field@"
],
"labels.alertname": [
"@opensearch-dashboards-highlighted-field@NeutronAgentDown@/opensearch-dashboards-highlighted-field@"
]
},
"sort": [
1783340569790
]
}
```