for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-cjz77bi3ugbaenpb45a5\/?after=9074467&quoteTargetID=UQ0_1&quoteRef=T332570\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_204\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9074467\" id=\"9074467\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9074467\" data-sigil=\"has-tooltip\" data-meta=\"0_202\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 3:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 15:53:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_200\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_1\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1078.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_213\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9074601\" id=\"9074601\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9074601\" data-sigil=\"has-tooltip\" data-meta=\"0_211\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 4:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 16:22:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_209\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_3\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1078.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1078 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308071558_btullis_872003_an-worker1078.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_222\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9074670\" id=\"9074670\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_221\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9074670\" data-sigil=\"has-tooltip\" data-meta=\"0_220\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 4:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 16:40:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_218\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_219\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_5\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1079.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_231\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9074821\" id=\"9074821\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_230\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9074821\" data-sigil=\"has-tooltip\" data-meta=\"0_229\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 5:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 17:22:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_227\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_7\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1079.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1079 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308071659_btullis_883331_an-worker1079.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_240\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9075384\" id=\"9075384\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9075384\" data-sigil=\"has-tooltip\" data-meta=\"0_238\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 8:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 20:21:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_236\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_9\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1080.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_249\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9075571\" id=\"9075571\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9075571\" data-sigil=\"has-tooltip\" data-meta=\"0_247\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 9:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 21:03:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_245\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_11\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1080.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1080 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308072041_btullis_925761_an-worker1080.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_258\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9075573\" id=\"9075573\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9075573\" data-sigil=\"has-tooltip\" data-meta=\"0_256\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 9:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 21:03:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_254\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_13\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1081.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_267\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9075672\" id=\"9075672\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_266\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9075672\" data-sigil=\"has-tooltip\" data-meta=\"0_265\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 9:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 21:43:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_263\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_15\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1081.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1081 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308072120_btullis_938210_an-worker1081.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_276\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/f7bqgojrukvox66675ks\/PHID-FILE-jbia46nhniqukzohp6zr\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BTullis\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9075750\" id=\"9075750\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_275\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BTullis\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003eBTullis\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9075750\" data-sigil=\"has-tooltip\" data-meta=\"0_274\"\u003e\u003cspan class=\"screen-only\"\u003eAug 7 2023, 10:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-07 22:12:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_272\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_17\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e12 of 79 Hadoop workers have now been upgraded.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ebtullis@cumin1001:~$ sudo cumin A:hadoop-worker 'cat \/etc\/debian_version'\n79 hosts will be targeted:\nan-worker[1078-1148].eqiad.wmnet,analytics[1070-1077].eqiad.wmnet\nOK to proceed on 79 hosts? Enter the number of affected hosts to confirm or "q" to quit: 79\n===== NODE GROUP =====                                                                                                                                                                                             \n(12) an-worker[1078-1081].eqiad.wmnet,analytics[1070-1077].eqiad.wmnet                                                                                                                                             \n----- OUTPUT of 'cat \/etc\/debian_version' -----                                                                                                                                                                    \n11.7                                                                                                                                                                                                               \n===== NODE GROUP =====                                                                                                                                                                                             \n(67) an-worker[1082-1148].eqiad.wmnet                                                                                                                                                                              \n----- OUTPUT of 'cat \/etc\/debian_version' -----                                                                                                                                                                    \n10.13                                                                                                                                                                                                              \n================                                                                                                                                                                                                   \nPASS |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 100% (79\/79) [00:01<00:00, 69.12hosts\/s]\nFAIL |                                                                                                                                                                            |   0% (0\/79) [00:01<?, ?hosts\/s]\n100.0% (79\/79) success ratio (>= 100.0% threshold) for command: 'cat \/etc\/debian_version'.\n100.0% (79\/79) success ratio (>= 100.0% threshold) of nodes successfully executed all commands.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAll of them have required some manual intervention so far. I'm hoping that this will reduce when we get toward the newer hosts.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_285\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9077352\" id=\"9077352\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_284\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9077352\" data-sigil=\"has-tooltip\" data-meta=\"0_283\"\u003e\u003cspan class=\"screen-only\"\u003eAug 8 2023, 2:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-08 14:03:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_281\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_19\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1082.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_294\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9077692\" id=\"9077692\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9077692\" data-sigil=\"has-tooltip\" data-meta=\"0_292\"\u003e\u003cspan class=\"screen-only\"\u003eAug 8 2023, 3:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-08 15:44:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_290\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_21\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1082.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1082 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308081522_btullis_1129991_an-worker1082.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_303\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9077714\" id=\"9077714\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_302\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9077714\" data-sigil=\"has-tooltip\" data-meta=\"0_301\"\u003e\u003cspan class=\"screen-only\"\u003eAug 8 2023, 3:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-08 15:53:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_299\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_23\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1083.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_312\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9078068\" id=\"9078068\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9078068\" data-sigil=\"has-tooltip\" data-meta=\"0_310\"\u003e\u003cspan class=\"screen-only\"\u003eAug 8 2023, 5:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-08 17:24:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_308\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_309\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1083.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1083 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308081701_btullis_1151032_an-worker1083.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_321\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9079548\" id=\"9079548\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9079548\" data-sigil=\"has-tooltip\" data-meta=\"0_319\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 9:54 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 09:54:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_317\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_318\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1084.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_330\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9079667\" id=\"9079667\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_329\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9079667\" data-sigil=\"has-tooltip\" data-meta=\"0_328\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 10:36 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 10:36:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_326\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_327\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_29\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1084.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1084 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308091012_btullis_1356753_an-worker1084.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_339\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9079776\" id=\"9079776\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_338\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9079776\" data-sigil=\"has-tooltip\" data-meta=\"0_337\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 11:20 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 11:20:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_335\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1085.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_348\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9079928\" id=\"9079928\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_347\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9079928\" data-sigil=\"has-tooltip\" data-meta=\"0_346\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 12:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 12:01:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_344\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_345\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1085.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1085 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308091138_btullis_1374384_an-worker1085.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_357\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9079931\" id=\"9079931\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9079931\" data-sigil=\"has-tooltip\" data-meta=\"0_355\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 12:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 12:01:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_353\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_354\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_35\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1086.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_366\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9080079\" id=\"9080079\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9080079\" data-sigil=\"has-tooltip\" data-meta=\"0_364\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 12:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 12:40:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_362\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_363\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1086.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1086 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308091217_btullis_1385306_an-worker1086.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_375\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9080330\" id=\"9080330\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9080330\" data-sigil=\"has-tooltip\" data-meta=\"0_373\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 1:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 13:12:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_371\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_372\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1087.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_384\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9080547\" id=\"9080547\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9080547\" data-sigil=\"has-tooltip\" data-meta=\"0_382\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 1:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 13:56:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_380\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1087.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1087 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308091332_btullis_1399747_an-worker1087.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_393\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9080669\" id=\"9080669\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9080669\" data-sigil=\"has-tooltip\" data-meta=\"0_391\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 2:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 14:18:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_389\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1088.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_402\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9080909\" id=\"9080909\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9080909\" data-sigil=\"has-tooltip\" data-meta=\"0_400\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 14:57:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_398\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1088.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1088 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308091434_btullis_1415057_an-worker1088.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_411\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9080916\" id=\"9080916\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9080916\" data-sigil=\"has-tooltip\" data-meta=\"0_409\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 14:57:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_407\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_408\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_47\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1089.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_420\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9081224\" id=\"9081224\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9081224\" data-sigil=\"has-tooltip\" data-meta=\"0_418\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 3:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 15:43:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_416\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_417\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1089.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1089 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308091520_btullis_1425271_an-worker1089.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_429\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/f7bqgojrukvox66675ks\/PHID-FILE-jbia46nhniqukzohp6zr\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BTullis\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9081585\" id=\"9081585\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BTullis\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003eBTullis\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9081585\" data-sigil=\"has-tooltip\" data-meta=\"0_427\"\u003e\u003cspan class=\"screen-only\"\u003eAug 9 2023, 5:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-09 17:30:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_425\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e59 workers remain to be upgraded.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ebtullis@cumin1001:~$ sudo cumin 'P{F:lsbdistcodename = buster} and A:hadoop-worker'\n59 hosts will be targeted:\nan-worker[1090-1148].eqiad.wmnet\nDRY-RUN mode enabled, aborting\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_438\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9083051\" id=\"9083051\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9083051\" data-sigil=\"has-tooltip\" data-meta=\"0_436\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 10:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 10:16:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_434\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1090.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_447\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9083145\" id=\"9083145\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_446\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9083145\" data-sigil=\"has-tooltip\" data-meta=\"0_445\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 10:55 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 10:55:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_443\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1090.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1090 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308101032_btullis_1641624_an-worker1090.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_456\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9083174\" id=\"9083174\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9083174\" data-sigil=\"has-tooltip\" data-meta=\"0_454\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 11:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 11:14:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_452\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1091.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_465\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9083332\" id=\"9083332\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_464\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9083332\" data-sigil=\"has-tooltip\" data-meta=\"0_463\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 12:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 12:08:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_461\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_462\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_59\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1091.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1091 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308101145_btullis_1654293_an-worker1091.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_474\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9083368\" id=\"9083368\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_473\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9083368\" data-sigil=\"has-tooltip\" data-meta=\"0_472\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 12:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 12:22:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_470\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_471\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1092.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_483\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9083616\" id=\"9083616\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_482\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9083616\" data-sigil=\"has-tooltip\" data-meta=\"0_481\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 1:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 13:22:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_479\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_480\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1092.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1092 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308101257_btullis_1671752_an-worker1092.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_486\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9084105\" id=\"9084105\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-times phui-timeline-icon\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003ejbond\u003c\/a\u003e unsubscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9084105\" data-sigil=\"has-tooltip\" data-meta=\"0_484\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2023, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-10 14:57:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_495\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9090448\" id=\"9090448\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_494\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9090448\" data-sigil=\"has-tooltip\" data-meta=\"0_493\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2023, 3:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-14 15:16:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_491\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_492\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_66\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1093.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_504\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9090600\" id=\"9090600\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_503\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9090600\" data-sigil=\"has-tooltip\" data-meta=\"0_502\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2023, 3:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-14 15:55:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_500\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_501\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_68\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1093.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1093 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308141532_btullis_2786742_an-worker1093.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_513\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9090607\" id=\"9090607\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_512\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9090607\" data-sigil=\"has-tooltip\" data-meta=\"0_511\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2023, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-14 15:58:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_509\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_510\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_70\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1094.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_522\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9090728\" id=\"9090728\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_521\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9090728\" data-sigil=\"has-tooltip\" data-meta=\"0_520\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2023, 4:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-14 16:37:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_518\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_519\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_72\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1094.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1094 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308141614_btullis_2797619_an-worker1094.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_531\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9090813\" id=\"9090813\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_530\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9090813\" data-sigil=\"has-tooltip\" data-meta=\"0_529\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2023, 5:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-14 17:02:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_527\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_528\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_74\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1095.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_540\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9090992\" id=\"9090992\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_539\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9090992\" data-sigil=\"has-tooltip\" data-meta=\"0_538\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2023, 5:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-14 17:41:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_536\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_537\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_76\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1095.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1095 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308141718_btullis_2811108_an-worker1095.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_549\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9092601\" id=\"9092601\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_548\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9092601\" data-sigil=\"has-tooltip\" data-meta=\"0_547\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2023, 9:34 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-15 09:34:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_545\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_546\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_78\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1096.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_558\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9092751\" id=\"9092751\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_557\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9092751\" data-sigil=\"has-tooltip\" data-meta=\"0_556\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2023, 10:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-15 10:56:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_554\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_555\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_80\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1096.eqiad.wmnet with OS bullseye executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1096 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_567\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9093278\" id=\"9093278\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9093278\" data-sigil=\"has-tooltip\" data-meta=\"0_565\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2023, 2:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-15 14:17:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_563\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1097.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_576\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9093472\" id=\"9093472\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_575\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9093472\" data-sigil=\"has-tooltip\" data-meta=\"0_574\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2023, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-15 15:07:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_572\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1097.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1097 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308151438_btullis_3066439_an-worker1097.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_585\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9093735\" id=\"9093735\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_584\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9093735\" data-sigil=\"has-tooltip\" data-meta=\"0_583\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2023, 4:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-15 16:05:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_581\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_582\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1098.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_594\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9093886\" id=\"9093886\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_593\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_87\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9093886\" data-sigil=\"has-tooltip\" data-meta=\"0_592\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2023, 4:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-15 16:49:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_590\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_591\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_88\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1098.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1098 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308151624_btullis_3094021_an-worker1098.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_603\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9095562\" id=\"9095562\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_602\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_89\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9095562\" data-sigil=\"has-tooltip\" data-meta=\"0_601\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 8:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 08:48:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_599\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_600\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_90\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1099.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_612\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9095702\" id=\"9095702\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_611\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9095702\" data-sigil=\"has-tooltip\" data-meta=\"0_610\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 9:29 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 09:29:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_608\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_609\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1099.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1099 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308160907_btullis_3291391_an-worker1099.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_621\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9095993\" id=\"9095993\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_620\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9095993\" data-sigil=\"has-tooltip\" data-meta=\"0_619\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 12:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 12:16:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_617\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_618\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1100.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_630\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/f7bqgojrukvox66675ks\/PHID-FILE-jbia46nhniqukzohp6zr\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BTullis\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096130\" id=\"9096130\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_629\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BTullis\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_95\"\u003eBTullis\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096130\" data-sigil=\"has-tooltip\" data-meta=\"0_628\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 12:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 12:56:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_626\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_627\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_96\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHappened upon a problem with an-worker1100, relating to the GPU support. This will probably require a patch.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@an-worker1100:~# puppet agent -tv\nInfo: Using configured environment 'production'\nInfo: Retrieving pluginfacts\nInfo: Retrieving plugin\nInfo: Retrieving locales\nInfo: Loading facts\nError: Could not retrieve catalog from remote server: Error 500 on SERVER: Server Error: Evaluation Error: Error while evaluating a Function Call, Please use ROCm 5.4 with Bullseye, other versions are not supported. (file: \/etc\/puppet\/modules\/amd_rocm\/manifests\/init.pp, line: 39, column: 9) on node an-worker1100.eqiad.wmnet\nWarning: Not using cache on failed catalog\nError: Could not retrieve catalog; skipping run\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_639\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096359\" id=\"9096359\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_638\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096359\" data-sigil=\"has-tooltip\" data-meta=\"0_637\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 1:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 13:42:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_635\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_636\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_98\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 949518 had a related patch set uploaded (by Btullis; author: Btullis):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] Remove the override for rocm version for buster hadoop workers\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/949518\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/949518\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_642\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9096360\" id=\"9096360\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_641\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_99\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096360\" data-sigil=\"has-tooltip\" data-meta=\"0_640\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 1:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 13:42:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_651\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096366\" id=\"9096366\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_650\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096366\" data-sigil=\"has-tooltip\" data-meta=\"0_649\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 1:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 13:43:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_647\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_648\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_102\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1101.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_660\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096369\" id=\"9096369\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_659\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_103\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096369\" data-sigil=\"has-tooltip\" data-meta=\"0_658\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 1:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 13:44:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_656\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_657\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_104\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 949518 \u003cstrong\u003emerged\u003c\/strong\u003e by Btullis:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] Remove the override for rocm version for buster hadoop workers\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/949518\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/949518\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_663\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9096545\" id=\"9096545\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_662\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_105\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096545\" data-sigil=\"has-tooltip\" data-meta=\"0_661\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 2:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 14:11:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_672\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096577\" id=\"9096577\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_671\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_107\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096577\" data-sigil=\"has-tooltip\" data-meta=\"0_670\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 2:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 14:23:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_668\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_669\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1100.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1100 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eFirst Puppet run failed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161233_btullis_3332281_an-worker1100.out, asking the operator what to do\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161412_btullis_3332281_an-worker1100.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_681\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096613\" id=\"9096613\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_680\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096613\" data-sigil=\"has-tooltip\" data-meta=\"0_679\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 2:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 14:30:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_677\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_110\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1101.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1101 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161403_btullis_3353401_an-worker1101.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_690\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096764\" id=\"9096764\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_689\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096764\" data-sigil=\"has-tooltip\" data-meta=\"0_688\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 3:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 15:00:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_686\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_687\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_112\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1102.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_699\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096765\" id=\"9096765\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_698\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_113\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096765\" data-sigil=\"has-tooltip\" data-meta=\"0_697\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 3:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 15:00:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_695\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_696\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_114\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1103.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_708\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096931\" id=\"9096931\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_707\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_115\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096931\" data-sigil=\"has-tooltip\" data-meta=\"0_706\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 3:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 15:41:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_704\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_705\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_116\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1103.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1103 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161517_btullis_3372258_an-worker1103.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_717\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9096936\" id=\"9096936\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_716\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_117\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9096936\" data-sigil=\"has-tooltip\" data-meta=\"0_715\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 3:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 15:43:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_713\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_118\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1102.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1102 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161519_btullis_3372218_an-worker1102.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_726\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9097049\" id=\"9097049\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_725\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_119\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9097049\" data-sigil=\"has-tooltip\" data-meta=\"0_724\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 4:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 16:23:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_722\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_723\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_120\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1104.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_735\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9097050\" id=\"9097050\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_734\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_121\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9097050\" data-sigil=\"has-tooltip\" data-meta=\"0_733\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 4:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 16:24:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_731\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_732\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_122\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1105.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_744\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9097230\" id=\"9097230\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_743\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_123\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9097230\" data-sigil=\"has-tooltip\" data-meta=\"0_742\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 5:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 17:04:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_740\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_741\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_124\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1104.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1104 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161641_btullis_3393584_an-worker1104.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_753\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9097234\" id=\"9097234\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_752\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9097234\" data-sigil=\"has-tooltip\" data-meta=\"0_751\"\u003e\u003cspan class=\"screen-only\"\u003eAug 16 2023, 5:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-16 17:06:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_749\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_750\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_126\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1105.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1105 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308161643_btullis_3393636_an-worker1105.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_762\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9098468\" id=\"9098468\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_761\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_127\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9098468\" data-sigil=\"has-tooltip\" data-meta=\"0_760\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2023, 9:28 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-17 09:28:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_758\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_759\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_128\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1106.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_771\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9098469\" id=\"9098469\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_770\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_129\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9098469\" data-sigil=\"has-tooltip\" data-meta=\"0_769\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2023, 9:29 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-17 09:29:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_767\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_768\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_130\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1107.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_780\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9098589\" id=\"9098589\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_779\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_131\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9098589\" data-sigil=\"has-tooltip\" data-meta=\"0_778\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2023, 10:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-17 10:09:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_776\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_777\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_132\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1107.eqiad.wmnet with OS bullseye executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1107 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eNew OS is buster but bullseye was requested\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_789\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9098590\" id=\"9098590\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_788\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_133\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9098590\" data-sigil=\"has-tooltip\" data-meta=\"0_787\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2023, 10:10 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-17 10:10:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_785\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_786\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1106.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1106 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308170946_btullis_3594346_an-worker1106.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_798\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9098596\" id=\"9098596\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_797\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_135\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9098596\" data-sigil=\"has-tooltip\" data-meta=\"0_796\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2023, 10:10 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-17 10:10:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_794\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_795\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_136\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by btullis@cumin1001 for host an-worker1107.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_807\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9098692\" id=\"9098692\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_806\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_137\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9098692\" data-sigil=\"has-tooltip\" data-meta=\"0_805\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2023, 10:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-17 10:49:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_803\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_804\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_138\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by btullis@cumin1001 for host an-worker1107.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1107 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308171026_btullis_3605379_an-worker1107.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_816\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/mjvddjhkjvorcvb3lxsg\/PHID-FILE-attrryu3mynzllzv6ulb\/5880b0-alphanumeric_lato-dark_S.png-0%2C0%2C0%2C0.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stevemunene\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105158\" id=\"9105158\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_815\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stevemunene\/\" class=\"phui-handle handle-status-closed handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_139\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Stevemunene\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105158\" data-sigil=\"has-tooltip\" data-meta=\"0_814\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 7:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 07:58:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_812\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_813\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_140\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ePicking this up from \u003ctt class=\"remarkup-monospaced\"\u003ean-worker1108\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_825\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105214\" id=\"9105214\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_824\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_141\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105214\" data-sigil=\"has-tooltip\" data-meta=\"0_823\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 8:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 08:26:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_821\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_822\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_142\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1108.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_834\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105362\" id=\"9105362\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_833\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_143\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105362\" data-sigil=\"has-tooltip\" data-meta=\"0_832\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 9:05 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 09:05:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_830\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_831\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_144\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1108.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1108 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308210844_stevemunene_472883_an-worker1108.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_843\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105514\" id=\"9105514\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_842\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_145\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105514\" data-sigil=\"has-tooltip\" data-meta=\"0_841\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 9:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 09:58:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_839\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_840\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_146\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1109.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_852\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105612\" id=\"9105612\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_851\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_147\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105612\" data-sigil=\"has-tooltip\" data-meta=\"0_850\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 10:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 10:33:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_848\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_849\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_148\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1110.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_861\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105640\" id=\"9105640\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_860\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_149\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105640\" data-sigil=\"has-tooltip\" data-meta=\"0_859\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 10:38 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 10:38:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_857\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_858\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1109.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1109 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308211015_stevemunene_493344_an-worker1109.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_870\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105721\" id=\"9105721\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_869\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_151\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105721\" data-sigil=\"has-tooltip\" data-meta=\"0_868\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 11:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 11:12:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_866\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_867\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_152\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1110.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1110 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308211051_stevemunene_502418_an-worker1110.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_879\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105918\" id=\"9105918\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_878\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_153\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105918\" data-sigil=\"has-tooltip\" data-meta=\"0_877\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 12:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 12:55:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_875\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_876\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_154\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1111.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_888\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9105922\" id=\"9105922\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_887\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_155\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9105922\" data-sigil=\"has-tooltip\" data-meta=\"0_886\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 12:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 12:56:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_884\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_885\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1112.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_897\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9106130\" id=\"9106130\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_896\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_157\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9106130\" data-sigil=\"has-tooltip\" data-meta=\"0_895\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 1:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 13:35:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_893\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_894\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_158\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1111.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1111 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308211313_stevemunene_538287_an-worker1111.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_906\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9106164\" id=\"9106164\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_905\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_159\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9106164\" data-sigil=\"has-tooltip\" data-meta=\"0_904\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2023, 1:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-21 13:39:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_902\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_903\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_160\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1112.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1112 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308211316_stevemunene_538396_an-worker1112.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_915\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108238\" id=\"9108238\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_914\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_161\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108238\" data-sigil=\"has-tooltip\" data-meta=\"0_913\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 5:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 05:37:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_911\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_912\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1113.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_924\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108239\" id=\"9108239\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_923\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_163\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108239\" data-sigil=\"has-tooltip\" data-meta=\"0_922\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 5:38 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 05:38:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_920\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_921\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_164\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1114.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_933\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108300\" id=\"9108300\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_932\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_165\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108300\" data-sigil=\"has-tooltip\" data-meta=\"0_931\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 6:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 06:17:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_929\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_930\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_166\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1113.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1113 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308220554_stevemunene_744330_an-worker1113.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_942\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108318\" id=\"9108318\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_941\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_167\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108318\" data-sigil=\"has-tooltip\" data-meta=\"0_940\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 6:21 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 06:21:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_938\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_939\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_168\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1114.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1114 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308220557_stevemunene_744539_an-worker1114.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_951\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108319\" id=\"9108319\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_950\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_169\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108319\" data-sigil=\"has-tooltip\" data-meta=\"0_949\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 6:21 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 06:21:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_947\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_948\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_170\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1115.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_960\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108354\" id=\"9108354\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_959\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_171\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108354\" data-sigil=\"has-tooltip\" data-meta=\"0_958\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 6:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 06:46:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_956\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_957\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_172\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1116.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_969\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108387\" id=\"9108387\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_968\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_173\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108387\" data-sigil=\"has-tooltip\" data-meta=\"0_967\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 7:05 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 07:05:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_965\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_966\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_174\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1115.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1115 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308220640_stevemunene_756308_an-worker1115.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_978\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108419\" id=\"9108419\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_977\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_175\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108419\" data-sigil=\"has-tooltip\" data-meta=\"0_976\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 7:27 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 07:27:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_974\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_975\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_176\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1116.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1116 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308220703_stevemunene_761937_an-worker1116.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_987\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9108440\" id=\"9108440\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_986\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_177\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108440\" data-sigil=\"has-tooltip\" data-meta=\"0_985\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 7:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 07:48:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_983\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_984\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_178\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1117.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_990\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wxni4dd2w73zpqvtmjdq\/PHID-FILE-ywrfhopc4oo74akgzhxh\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gehel\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9108506\" id=\"9108506\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_989\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gehel\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_179\"\u003eGehel\u003c\/a\u003e reassigned this task from \u003ca href=\"\/p\/BTullis\/\" class=\"phui-handle phui-link-person\"\u003eBTullis\u003c\/a\u003e to \u003ca href=\"\/p\/Stevemunene\/\" class=\"phui-handle handle-status-closed handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Stevemunene\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9108506\" data-sigil=\"has-tooltip\" data-meta=\"0_988\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 8:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 08:19:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_999\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/mjvddjhkjvorcvb3lxsg\/PHID-FILE-attrryu3mynzllzv6ulb\/5880b0-alphanumeric_lato-dark_S.png-0%2C0%2C0%2C0.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stevemunene\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9109031\" id=\"9109031\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_998\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stevemunene\/\" class=\"phui-handle handle-status-closed handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Stevemunene\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9109031\" data-sigil=\"has-tooltip\" data-meta=\"0_997\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 12:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 12:01:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_995\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_996\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWhile reimaging \u003ctt class=\"remarkup-monospaced\"\u003ean-worker1117.eqiad.wmnet\u003c\/tt\u003e we found that the server did not reimage with the right mountpoints resulting in puppet error\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eError while evaluating a Function Call:\nNumber of datanode mountpoints (0) below threshold: 10, please check.\n\u2026in \/etc\/puppet\/modules\/profile\/manifests\/hadoop\/common.pp, line: 389, column: 9.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eUsing findmnt we can confirm that the required datanode mountpoints are indeed unavailable.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ean-worker1117:~# findmnt\nTARGET                       SOURCE                              FSTYPE     OPTIONS\n\/                            \/dev\/mapper\/an--worker1117--vg-root ext4       rw,relatime,errors=remount-ro\n\u251c\u2500\/sys                       sysfs                               sysfs      rw,nosuid,nodev,noexec,relatime\n\u2502 \u251c\u2500\/sys\/kernel\/security     securityfs                          securityfs rw,nosuid,nodev,noexec,relatime\n\u2502 \u251c\u2500\/sys\/fs\/cgroup           cgroup2                             cgroup2    rw,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot\n\u2502 \u251c\u2500\/sys\/fs\/pstore           pstore                              pstore     rw,nosuid,nodev,noexec,relatime\n\u2502 \u251c\u2500\/sys\/fs\/bpf              none                                bpf        rw,nosuid,nodev,noexec,relatime,mode=700\n\u2502 \u251c\u2500\/sys\/kernel\/debug        debugfs                             debugfs    rw,nosuid,nodev,noexec,relatime\n\u2502 \u251c\u2500\/sys\/kernel\/tracing      tracefs                             tracefs    rw,nosuid,nodev,noexec,relatime\n\u2502 \u251c\u2500\/sys\/kernel\/config       configfs                            configfs   rw,nosuid,nodev,noexec,relatime\n\u2502 \u2514\u2500\/sys\/fs\/fuse\/connections fusectl                             fusectl    rw,nosuid,nodev,noexec,relatime\n\u251c\u2500\/proc                      proc                                proc       rw,nosuid,nodev,noexec,relatime\n\u2502 \u2514\u2500\/proc\/sys\/fs\/binfmt_misc systemd-1                           autofs     rw,relatime,fd=30,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=80142\n\u251c\u2500\/dev                       udev                                devtmpfs   rw,nosuid,relatime,size=98296276k,nr_inodes=24574069,mode=755\n\u2502 \u251c\u2500\/dev\/pts                 devpts                              devpts     rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000\n\u2502 \u251c\u2500\/dev\/shm                 tmpfs                               tmpfs      rw,nosuid,nodev\n\u2502 \u251c\u2500\/dev\/hugepages           hugetlbfs                           hugetlbfs  rw,relatime,pagesize=2M\n\u2502 \u2514\u2500\/dev\/mqueue              mqueue                              mqueue     rw,nosuid,nodev,noexec,relatime\n\u251c\u2500\/run                       tmpfs                               tmpfs      rw,nosuid,nodev,noexec,relatime,size=19663116k,mode=755\n\u2502 \u251c\u2500\/run\/lock                tmpfs                               tmpfs      rw,nosuid,nodev,noexec,relatime,size=5120k\n\u2502 \u2514\u2500\/run\/user\/0              tmpfs                               tmpfs      rw,nosuid,nodev,relatime,size=19663112k,nr_inodes=4915778,mode=700\n\u2514\u2500\/boot                      \/dev\/sda1                           ext4       rw,relatime\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThis is due to the fact that \u003ca href=\"https:\/\/github.com\/wikimedia\/operations-puppet\/blob\/production\/modules\/install_server\/files\/autoinstall\/netboot.cfg#L99\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ean-worker1117\u003c\/a\u003e has never been reimaged and are configured to use the first install partman recipe and not the reinstall recipe. This also includes the servers \u003ctt class=\"remarkup-monospaced\"\u003ean-worker111[7-9]*|an-worker112*|an-worker113*|an-worker114[0-8]\u003c\/tt\u003e so we shall be moving them to the \u003ctt class=\"remarkup-monospaced\"\u003ereuse-analytics-hadoop-worker-12dev.cfg\u003c\/tt\u003e recipe. This should be covered by the \u003ca href=\"https:\/\/github.com\/wikimedia\/operations-puppet\/blob\/cac36deb4e98fb9f27f3ede110f92766628f6be4\/modules\/install_server\/files\/autoinstall\/netboot.cfg#L104C45-L104C45\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ean-worker*\u003c\/a\u003e expression.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1008\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9109042\" id=\"9109042\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1007\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9109042\" data-sigil=\"has-tooltip\" data-meta=\"0_1006\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 12:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 12:02:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1004\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1005\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_183\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 951458 had a related patch set uploaded (by Stevemunene; author: Stevemunene):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] switch an-worker[17-48] to reuse-analytics-hadoop recipe\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/951458\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/951458\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1011\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9109043\" id=\"9109043\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_1010\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_184\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_185\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9109043\" data-sigil=\"has-tooltip\" data-meta=\"0_1009\"\u003e\u003cspan class=\"screen-only\"\u003eAug 22 2023, 12:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-22 12:02:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1020\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9113688\" id=\"9113688\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1019\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_186\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9113688\" data-sigil=\"has-tooltip\" data-meta=\"0_1018\"\u003e\u003cspan class=\"screen-only\"\u003eAug 23 2023, 2:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-23 14:15:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1016\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1017\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_187\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 951458 \u003cstrong\u003emerged\u003c\/strong\u003e by Stevemunene:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] switch an-worker[17-48] to reuse-analytics-hadoop recipe\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/951458\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/951458\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1029\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9113690\" id=\"9113690\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1028\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_188\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9113690\" data-sigil=\"has-tooltip\" data-meta=\"0_1027\"\u003e\u003cspan class=\"screen-only\"\u003eAug 23 2023, 2:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-23 14:16:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1025\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1026\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1117.eqiad.wmnet with OS bullseye executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1117 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eFirst Puppet run failed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308220805_stevemunene_776100_an-worker1117.out, asking the operator what to do\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eFirst Puppet run failed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202308220929_stevemunene_776100_an-worker1117.out, asking the operator what to do\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eFirst Puppet run failed and the operator aborted\u003c\/strong\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1032\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9113713\" id=\"9113713\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_1031\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_190\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9113713\" data-sigil=\"has-tooltip\" data-meta=\"0_1030\"\u003e\u003cspan class=\"screen-only\"\u003eAug 23 2023, 2:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-23 14:29:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1041\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9113850\" id=\"9113850\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1040\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_192\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9113850\" data-sigil=\"has-tooltip\" data-meta=\"0_1039\"\u003e\u003cspan class=\"screen-only\"\u003eAug 23 2023, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-23 14:57:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1037\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1038\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_193\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by stevemunene@cumin1001 for host an-worker1117.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1050\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9114023\" id=\"9114023\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1049\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9114023\" data-sigil=\"has-tooltip\" data-meta=\"0_1048\"\u003e\u003cspan class=\"screen-only\"\u003eAug 23 2023, 3:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-08-23 15:44:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1046\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1047\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_195\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by stevemunene@cumin1001 for host an-worker1117.eqiad.wmnet with OS bullseye executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ean-worker1117 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eUnable to disable Puppet, the host may have been unreachable\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDeleted any existing Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fjiihickroeli5r"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-xamdj5lfdddjafm"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-4rp5pgik4otvn6l"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-uz42wz4at7x4gss"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-pn6cve6drokil4p"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-tvxqmkibpmra7cy"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-r5ljup76ydtalhv"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-e6p3i5pcrhjm6rh"},{"hovercardSpec":{"objectPHID":"PHID-USER-7bznjrywy72euasp5sjz"}},{"phid":"PHID-XACT-TASK-2xmarrd6hklciit"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-bq3hot73rs33hcn"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ynutsusirbrapsd"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-bgmjkkdrpcgjh5b"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-cj4u54t7cd3cgvj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-vj2qwmb7wgfhf74"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-5rlug3b6w4nlp5m"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-o52c3p4754dwnab"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-hqekgq6xwnjlouk"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-c64vqg5neelqhi3"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ft4ix5ciwchkvfi"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-iyxlsbmip7e2enb"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3rp4kykvzp56fv4"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3d2q7s2lssbk3vw"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-h2dbzjbhqv4c4i4"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ijc7cxychg5s2il"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fpxtjjivxenykit"},{"hovercardSpec":{"objectPHID":"PHID-USER-7bznjrywy72euasp5sjz"}},{"phid":"PHID-XACT-TASK-upwhs3foodmwgeu"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-va4u4fsr6t6iigl"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-obxnacn6hl5sujj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-asnkucfdc25zjvr"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3zrdwjzkaqzngbp"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-esya5dpaaijltzb"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-n3sidkqoat6vkk4"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-yucik5igukl2ql6"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ugyjuojdne2bu65"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-jouxpdlklbnlt4p"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ugcfq3z3dejfmmt"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-t72wk3zl7isnzd7"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-cmdr6ukcps3nguj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-hddpmnjj7spvqkg"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-r46doyg5q3lf256"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-wsparfznnofzqtw"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-rt57whvy3jxga5f"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2efbjpdzxnrxtxh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-yb7o3qepi5kzjgk"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-gvihl34w7xnsufd"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-qncxamzuj2kz7yc"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-p7durllr4wz346u"},{"hovercardSpec":{"objectPHID":"PHID-USER-7bznjrywy72euasp5sjz"}},{"phid":"PHID-XACT-TASK-klluty3s5uhw56v"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-vik4ekegkq55khr"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3apad2umml7shr6"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-sx4qb5rvlrl6eii"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-caqghwodxaf647u"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-5qycgwaqbkath32"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3o4g3m5xasl3a7d"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-vrxq3huul2awo2x"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-lkzyj7dnxeasq63"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-wlefaj5u4ojtf2k"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-upuowb4r3z6iziv"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2c2dtmkcyflzosl"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-l7jrp24axllbg2w"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-i5uotexlveatqwi"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-32bizmv5zwp3vr2"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-qnc2mr5d7a2bdsg"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-gk4lf5at6q73noh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ebzs7j32tzkmizc"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-c4jq7ed7m6bh6nh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-txhgcxw4eacfx7z"},{"hovercardSpec":{"objectPHID":"PHID-USER-jc2itqyiw722xlf52zrt"}},{"phid":"PHID-XACT-TASK-aakeh2awdwpgewz"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-moswqgrdkeqii7z"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ctuvtlsxiizj45f"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ej3nmlyi733go2h"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-6ydhx5hkdkpi4wu"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-medkhb34673wozy"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fue2cbzogwvod7c"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-5si6vrfteasexob"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-pwmjbj2fm3wk5dp"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-xjqu4jwoxd4k26a"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kklazu2w5ojfvsu"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-6pox5l2s5pefbfm"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-dh4r6vin2hu2pol"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2z2y5t3ugqfc62r"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-laxj7mjmwmsatbh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-av2dnb6y4bbx3yi"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ueojrl2rxsvg6lx"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-5i2qdwqlduh5nce"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-xejrpfr65gypm37"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-hfpnepgqapj3oje"},{"hovercardSpec":{"objectPHID":"PHID-USER-oknilgmxl4rosyulvny7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jc2itqyiw722xlf52zrt"}},{"phid":"PHID-XACT-TASK-fzvdai4edk5dwzv"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-czgjanuqflyut2k"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-dvqyk2xt7fpmgsq"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2ndx2gj2uqyqvlf"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-jyqqf32oz4ijrqs"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-tntne2obemjs6vs"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fjiihickroeli5r\/","ref":"T332570#9074467"},[],{"anchor":"9074467"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_196\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fjiihickroeli5r\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_198\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_199\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fjiihickroeli5r","anchor":"9074467"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xamdj5lfdddjafm\/","ref":"T332570#9074601"},[],{"anchor":"9074601"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_205\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_206\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xamdj5lfdddjafm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_207\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_208\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xamdj5lfdddjafm","anchor":"9074601"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4rp5pgik4otvn6l\/","ref":"T332570#9074670"},[],{"anchor":"9074670"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_214\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_215\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4rp5pgik4otvn6l\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_216\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_217\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4rp5pgik4otvn6l","anchor":"9074670"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uz42wz4at7x4gss\/","ref":"T332570#9074821"},[],{"anchor":"9074821"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_223\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_224\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uz42wz4at7x4gss\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_225\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_226\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uz42wz4at7x4gss","anchor":"9074821"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pn6cve6drokil4p\/","ref":"T332570#9075384"},[],{"anchor":"9075384"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_232\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pn6cve6drokil4p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_234\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_235\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pn6cve6drokil4p","anchor":"9075384"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tvxqmkibpmra7cy\/","ref":"T332570#9075571"},[],{"anchor":"9075571"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_241\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tvxqmkibpmra7cy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_243\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_244\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-tvxqmkibpmra7cy","anchor":"9075571"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-r5ljup76ydtalhv\/","ref":"T332570#9075573"},[],{"anchor":"9075573"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_250\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r5ljup76ydtalhv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-r5ljup76ydtalhv","anchor":"9075573"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-e6p3i5pcrhjm6rh\/","ref":"T332570#9075672"},[],{"anchor":"9075672"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_259\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-e6p3i5pcrhjm6rh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_261\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-e6p3i5pcrhjm6rh","anchor":"9075672"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2xmarrd6hklciit\/","ref":"T332570#9075750"},[],{"anchor":"9075750"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_268\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2xmarrd6hklciit\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_270\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2xmarrd6hklciit","anchor":"9075750"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bq3hot73rs33hcn\/","ref":"T332570#9077352"},[],{"anchor":"9077352"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_277\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bq3hot73rs33hcn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_279\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bq3hot73rs33hcn","anchor":"9077352"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ynutsusirbrapsd\/","ref":"T332570#9077692"},[],{"anchor":"9077692"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_286\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ynutsusirbrapsd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_288\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ynutsusirbrapsd","anchor":"9077692"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bgmjkkdrpcgjh5b\/","ref":"T332570#9077714"},[],{"anchor":"9077714"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_295\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bgmjkkdrpcgjh5b\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_297\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bgmjkkdrpcgjh5b","anchor":"9077714"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cj4u54t7cd3cgvj\/","ref":"T332570#9078068"},[],{"anchor":"9078068"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_304\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cj4u54t7cd3cgvj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_306\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cj4u54t7cd3cgvj","anchor":"9078068"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vj2qwmb7wgfhf74\/","ref":"T332570#9079548"},[],{"anchor":"9079548"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_313\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vj2qwmb7wgfhf74\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_315\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vj2qwmb7wgfhf74","anchor":"9079548"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5rlug3b6w4nlp5m\/","ref":"T332570#9079667"},[],{"anchor":"9079667"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_322\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5rlug3b6w4nlp5m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_324\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5rlug3b6w4nlp5m","anchor":"9079667"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-o52c3p4754dwnab\/","ref":"T332570#9079776"},[],{"anchor":"9079776"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_331\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-o52c3p4754dwnab\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_333\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-o52c3p4754dwnab","anchor":"9079776"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hqekgq6xwnjlouk\/","ref":"T332570#9079928"},[],{"anchor":"9079928"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_340\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hqekgq6xwnjlouk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_342\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-hqekgq6xwnjlouk","anchor":"9079928"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-c64vqg5neelqhi3\/","ref":"T332570#9079931"},[],{"anchor":"9079931"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_349\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-c64vqg5neelqhi3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_351\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-c64vqg5neelqhi3","anchor":"9079931"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ft4ix5ciwchkvfi\/","ref":"T332570#9080079"},[],{"anchor":"9080079"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_358\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ft4ix5ciwchkvfi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_360\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_361\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ft4ix5ciwchkvfi","anchor":"9080079"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-iyxlsbmip7e2enb\/","ref":"T332570#9080330"},[],{"anchor":"9080330"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_367\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-iyxlsbmip7e2enb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_369\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_370\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-iyxlsbmip7e2enb","anchor":"9080330"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3rp4kykvzp56fv4\/","ref":"T332570#9080547"},[],{"anchor":"9080547"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_376\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3rp4kykvzp56fv4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_378\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3rp4kykvzp56fv4","anchor":"9080547"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3d2q7s2lssbk3vw\/","ref":"T332570#9080669"},[],{"anchor":"9080669"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_385\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3d2q7s2lssbk3vw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_387\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3d2q7s2lssbk3vw","anchor":"9080669"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-h2dbzjbhqv4c4i4\/","ref":"T332570#9080909"},[],{"anchor":"9080909"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_394\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_395\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-h2dbzjbhqv4c4i4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_396\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_397\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-h2dbzjbhqv4c4i4","anchor":"9080909"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ijc7cxychg5s2il\/","ref":"T332570#9080916"},[],{"anchor":"9080916"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_403\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ijc7cxychg5s2il\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_405\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_406\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ijc7cxychg5s2il","anchor":"9080916"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fpxtjjivxenykit\/","ref":"T332570#9081224"},[],{"anchor":"9081224"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_412\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_413\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fpxtjjivxenykit\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_414\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_415\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fpxtjjivxenykit","anchor":"9081224"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-upwhs3foodmwgeu\/","ref":"T332570#9081585"},[],{"anchor":"9081585"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_421\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_422\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-upwhs3foodmwgeu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_423\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-upwhs3foodmwgeu","anchor":"9081585"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-va4u4fsr6t6iigl\/","ref":"T332570#9083051"},[],{"anchor":"9083051"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_430\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-va4u4fsr6t6iigl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_432\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-va4u4fsr6t6iigl","anchor":"9083051"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-obxnacn6hl5sujj\/","ref":"T332570#9083145"},[],{"anchor":"9083145"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_439\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-obxnacn6hl5sujj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_441\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-obxnacn6hl5sujj","anchor":"9083145"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-asnkucfdc25zjvr\/","ref":"T332570#9083174"},[],{"anchor":"9083174"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_448\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-asnkucfdc25zjvr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_450\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-asnkucfdc25zjvr","anchor":"9083174"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3zrdwjzkaqzngbp\/","ref":"T332570#9083332"},[],{"anchor":"9083332"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_457\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3zrdwjzkaqzngbp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_459\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_460\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3zrdwjzkaqzngbp","anchor":"9083332"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-esya5dpaaijltzb\/","ref":"T332570#9083368"},[],{"anchor":"9083368"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_466\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_467\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-esya5dpaaijltzb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_468\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_469\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-esya5dpaaijltzb","anchor":"9083368"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-n3sidkqoat6vkk4\/","ref":"T332570#9083616"},[],{"anchor":"9083616"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_475\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_476\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-n3sidkqoat6vkk4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_477\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_478\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-n3sidkqoat6vkk4","anchor":"9083616"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uu4d4jytpy6vjko","anchor":"9084105"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yucik5igukl2ql6\/","ref":"T332570#9090448"},[],{"anchor":"9090448"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_487\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_488\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yucik5igukl2ql6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_489\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_490\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-yucik5igukl2ql6","anchor":"9090448"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ugyjuojdne2bu65\/","ref":"T332570#9090600"},[],{"anchor":"9090600"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_496\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_497\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ugyjuojdne2bu65\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_498\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_499\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ugyjuojdne2bu65","anchor":"9090600"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jouxpdlklbnlt4p\/","ref":"T332570#9090607"},[],{"anchor":"9090607"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_505\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_506\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jouxpdlklbnlt4p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_507\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_508\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jouxpdlklbnlt4p","anchor":"9090607"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ugcfq3z3dejfmmt\/","ref":"T332570#9090728"},[],{"anchor":"9090728"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_514\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_515\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ugcfq3z3dejfmmt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_516\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_517\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ugcfq3z3dejfmmt","anchor":"9090728"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-t72wk3zl7isnzd7\/","ref":"T332570#9090813"},[],{"anchor":"9090813"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_523\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_524\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t72wk3zl7isnzd7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_525\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_526\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-t72wk3zl7isnzd7","anchor":"9090813"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cmdr6ukcps3nguj\/","ref":"T332570#9090992"},[],{"anchor":"9090992"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_532\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_533\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cmdr6ukcps3nguj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_534\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_535\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cmdr6ukcps3nguj","anchor":"9090992"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hddpmnjj7spvqkg\/","ref":"T332570#9092601"},[],{"anchor":"9092601"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_541\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_542\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hddpmnjj7spvqkg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_543\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_544\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-hddpmnjj7spvqkg","anchor":"9092601"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-r46doyg5q3lf256\/","ref":"T332570#9092751"},[],{"anchor":"9092751"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_550\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_551\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r46doyg5q3lf256\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_552\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_553\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-r46doyg5q3lf256","anchor":"9092751"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wsparfznnofzqtw\/","ref":"T332570#9093278"},[],{"anchor":"9093278"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_559\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_560\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wsparfznnofzqtw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_561\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_562\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wsparfznnofzqtw","anchor":"9093278"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rt57whvy3jxga5f\/","ref":"T332570#9093472"},[],{"anchor":"9093472"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_568\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_569\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rt57whvy3jxga5f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_570\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_571\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rt57whvy3jxga5f","anchor":"9093472"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2efbjpdzxnrxtxh\/","ref":"T332570#9093735"},[],{"anchor":"9093735"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_577\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_578\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2efbjpdzxnrxtxh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_579\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_580\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2efbjpdzxnrxtxh","anchor":"9093735"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yb7o3qepi5kzjgk\/","ref":"T332570#9093886"},[],{"anchor":"9093886"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_586\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_587\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yb7o3qepi5kzjgk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_588\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_589\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-yb7o3qepi5kzjgk","anchor":"9093886"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gvihl34w7xnsufd\/","ref":"T332570#9095562"},[],{"anchor":"9095562"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_595\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_596\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gvihl34w7xnsufd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_597\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_598\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gvihl34w7xnsufd","anchor":"9095562"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qncxamzuj2kz7yc\/","ref":"T332570#9095702"},[],{"anchor":"9095702"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_604\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_605\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qncxamzuj2kz7yc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_606\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_607\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-qncxamzuj2kz7yc","anchor":"9095702"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-p7durllr4wz346u\/","ref":"T332570#9095993"},[],{"anchor":"9095993"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_613\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_614\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-p7durllr4wz346u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_615\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_616\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-p7durllr4wz346u","anchor":"9095993"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-klluty3s5uhw56v\/","ref":"T332570#9096130"},[],{"anchor":"9096130"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_622\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_623\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-klluty3s5uhw56v\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_624\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_625\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-klluty3s5uhw56v","anchor":"9096130"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vik4ekegkq55khr\/","ref":"T332570#9096359"},[],{"anchor":"9096359"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_631\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_632\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vik4ekegkq55khr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_633\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_634\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vik4ekegkq55khr","anchor":"9096359"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-hon6orwkjasjax6","anchor":"9096360"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3apad2umml7shr6\/","ref":"T332570#9096366"},[],{"anchor":"9096366"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_643\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_644\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3apad2umml7shr6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_645\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_646\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3apad2umml7shr6","anchor":"9096366"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-sx4qb5rvlrl6eii\/","ref":"T332570#9096369"},[],{"anchor":"9096369"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_652\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_653\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sx4qb5rvlrl6eii\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_654\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_655\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-sx4qb5rvlrl6eii","anchor":"9096369"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-qy6wzkn52jjmlj4","anchor":"9096545"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-caqghwodxaf647u\/","ref":"T332570#9096577"},[],{"anchor":"9096577"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_664\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_665\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-caqghwodxaf647u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_666\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_667\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-caqghwodxaf647u","anchor":"9096577"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5qycgwaqbkath32\/","ref":"T332570#9096613"},[],{"anchor":"9096613"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_673\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_674\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5qycgwaqbkath32\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_675\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_676\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5qycgwaqbkath32","anchor":"9096613"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3o4g3m5xasl3a7d\/","ref":"T332570#9096764"},[],{"anchor":"9096764"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_682\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_683\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_215\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3o4g3m5xasl3a7d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_684\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_685\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3o4g3m5xasl3a7d","anchor":"9096764"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vrxq3huul2awo2x\/","ref":"T332570#9096765"},[],{"anchor":"9096765"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_217\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_691\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_692\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_219\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vrxq3huul2awo2x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_693\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_694\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vrxq3huul2awo2x","anchor":"9096765"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lkzyj7dnxeasq63\/","ref":"T332570#9096931"},[],{"anchor":"9096931"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_221\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_700\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_701\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_223\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lkzyj7dnxeasq63\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_702\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_703\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-lkzyj7dnxeasq63","anchor":"9096931"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wlefaj5u4ojtf2k\/","ref":"T332570#9096936"},[],{"anchor":"9096936"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_225\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_709\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_710\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_227\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wlefaj5u4ojtf2k\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_711\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_712\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wlefaj5u4ojtf2k","anchor":"9096936"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-upuowb4r3z6iziv\/","ref":"T332570#9097049"},[],{"anchor":"9097049"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_229\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_718\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_719\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_231\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-upuowb4r3z6iziv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_720\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_721\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-upuowb4r3z6iziv","anchor":"9097049"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2c2dtmkcyflzosl\/","ref":"T332570#9097050"},[],{"anchor":"9097050"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_233\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_727\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_728\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_235\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2c2dtmkcyflzosl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_729\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_730\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2c2dtmkcyflzosl","anchor":"9097050"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-l7jrp24axllbg2w\/","ref":"T332570#9097230"},[],{"anchor":"9097230"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_237\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_736\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_737\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_239\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l7jrp24axllbg2w\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_738\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_739\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-l7jrp24axllbg2w","anchor":"9097230"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-i5uotexlveatqwi\/","ref":"T332570#9097234"},[],{"anchor":"9097234"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_241\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_745\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_746\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_243\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-i5uotexlveatqwi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_747\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_748\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-i5uotexlveatqwi","anchor":"9097234"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-32bizmv5zwp3vr2\/","ref":"T332570#9098468"},[],{"anchor":"9098468"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_245\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_754\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_755\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_247\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-32bizmv5zwp3vr2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_756\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_757\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-32bizmv5zwp3vr2","anchor":"9098468"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qnc2mr5d7a2bdsg\/","ref":"T332570#9098469"},[],{"anchor":"9098469"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_249\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_763\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_764\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_251\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qnc2mr5d7a2bdsg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_765\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_766\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-qnc2mr5d7a2bdsg","anchor":"9098469"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gk4lf5at6q73noh\/","ref":"T332570#9098589"},[],{"anchor":"9098589"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_253\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_772\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_773\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_255\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gk4lf5at6q73noh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_774\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_775\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gk4lf5at6q73noh","anchor":"9098589"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ebzs7j32tzkmizc\/","ref":"T332570#9098590"},[],{"anchor":"9098590"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_257\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_781\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_782\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_259\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ebzs7j32tzkmizc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_783\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_784\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ebzs7j32tzkmizc","anchor":"9098590"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-c4jq7ed7m6bh6nh\/","ref":"T332570#9098596"},[],{"anchor":"9098596"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_261\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_790\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_791\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_263\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-c4jq7ed7m6bh6nh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_792\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_793\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-c4jq7ed7m6bh6nh","anchor":"9098596"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-txhgcxw4eacfx7z\/","ref":"T332570#9098692"},[],{"anchor":"9098692"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_265\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_799\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_800\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_267\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-txhgcxw4eacfx7z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_801\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_802\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-txhgcxw4eacfx7z","anchor":"9098692"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-aakeh2awdwpgewz\/","ref":"T332570#9105158"},[],{"anchor":"9105158"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_269\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_808\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_809\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_271\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aakeh2awdwpgewz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_810\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_811\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-aakeh2awdwpgewz","anchor":"9105158"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-moswqgrdkeqii7z\/","ref":"T332570#9105214"},[],{"anchor":"9105214"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_273\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_817\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_818\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_275\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-moswqgrdkeqii7z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_819\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_820\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-moswqgrdkeqii7z","anchor":"9105214"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ctuvtlsxiizj45f\/","ref":"T332570#9105362"},[],{"anchor":"9105362"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_277\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_826\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_827\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_279\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ctuvtlsxiizj45f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_828\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_829\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ctuvtlsxiizj45f","anchor":"9105362"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ej3nmlyi733go2h\/","ref":"T332570#9105514"},[],{"anchor":"9105514"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_281\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_835\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_836\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_283\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ej3nmlyi733go2h\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_837\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_838\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ej3nmlyi733go2h","anchor":"9105514"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6ydhx5hkdkpi4wu\/","ref":"T332570#9105612"},[],{"anchor":"9105612"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_285\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_844\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_845\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_287\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6ydhx5hkdkpi4wu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_846\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_847\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6ydhx5hkdkpi4wu","anchor":"9105612"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-medkhb34673wozy\/","ref":"T332570#9105640"},[],{"anchor":"9105640"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_289\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_853\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_854\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_291\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-medkhb34673wozy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_855\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_856\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-medkhb34673wozy","anchor":"9105640"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fue2cbzogwvod7c\/","ref":"T332570#9105721"},[],{"anchor":"9105721"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_293\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_862\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_863\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_295\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fue2cbzogwvod7c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_864\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_865\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fue2cbzogwvod7c","anchor":"9105721"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5si6vrfteasexob\/","ref":"T332570#9105918"},[],{"anchor":"9105918"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_297\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_871\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_872\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_299\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5si6vrfteasexob\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_873\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_874\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5si6vrfteasexob","anchor":"9105918"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pwmjbj2fm3wk5dp\/","ref":"T332570#9105922"},[],{"anchor":"9105922"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_301\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_880\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_881\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_303\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pwmjbj2fm3wk5dp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_882\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_883\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pwmjbj2fm3wk5dp","anchor":"9105922"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xjqu4jwoxd4k26a\/","ref":"T332570#9106130"},[],{"anchor":"9106130"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_305\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_889\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_890\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_307\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xjqu4jwoxd4k26a\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_891\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_892\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xjqu4jwoxd4k26a","anchor":"9106130"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kklazu2w5ojfvsu\/","ref":"T332570#9106164"},[],{"anchor":"9106164"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_309\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_898\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_899\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_311\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kklazu2w5ojfvsu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_900\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_901\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kklazu2w5ojfvsu","anchor":"9106164"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6pox5l2s5pefbfm\/","ref":"T332570#9108238"},[],{"anchor":"9108238"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_313\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_907\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_908\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_315\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6pox5l2s5pefbfm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_909\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_910\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6pox5l2s5pefbfm","anchor":"9108238"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dh4r6vin2hu2pol\/","ref":"T332570#9108239"},[],{"anchor":"9108239"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_317\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_916\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_917\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_319\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dh4r6vin2hu2pol\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_918\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_919\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dh4r6vin2hu2pol","anchor":"9108239"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2z2y5t3ugqfc62r\/","ref":"T332570#9108300"},[],{"anchor":"9108300"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_321\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_925\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_926\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_323\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2z2y5t3ugqfc62r\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_927\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_928\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2z2y5t3ugqfc62r","anchor":"9108300"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-laxj7mjmwmsatbh\/","ref":"T332570#9108318"},[],{"anchor":"9108318"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_325\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_934\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_935\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_327\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-laxj7mjmwmsatbh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_936\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_937\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-laxj7mjmwmsatbh","anchor":"9108318"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-av2dnb6y4bbx3yi\/","ref":"T332570#9108319"},[],{"anchor":"9108319"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_329\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_943\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_944\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_331\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-av2dnb6y4bbx3yi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_945\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_946\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-av2dnb6y4bbx3yi","anchor":"9108319"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ueojrl2rxsvg6lx\/","ref":"T332570#9108354"},[],{"anchor":"9108354"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_333\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_952\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_953\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_335\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ueojrl2rxsvg6lx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_954\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_955\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ueojrl2rxsvg6lx","anchor":"9108354"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5i2qdwqlduh5nce\/","ref":"T332570#9108387"},[],{"anchor":"9108387"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_337\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_961\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_962\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_339\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5i2qdwqlduh5nce\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_963\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_964\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5i2qdwqlduh5nce","anchor":"9108387"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xejrpfr65gypm37\/","ref":"T332570#9108419"},[],{"anchor":"9108419"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_341\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_970\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_971\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_343\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xejrpfr65gypm37\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_972\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_973\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xejrpfr65gypm37","anchor":"9108419"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hfpnepgqapj3oje\/","ref":"T332570#9108440"},[],{"anchor":"9108440"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_345\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_979\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_980\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_347\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hfpnepgqapj3oje\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_981\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_982\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-hfpnepgqapj3oje","anchor":"9108440"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tv256exuzpcxt37","anchor":"9108506"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fzvdai4edk5dwzv\/","ref":"T332570#9109031"},[],{"anchor":"9109031"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_349\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_991\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_992\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_351\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fzvdai4edk5dwzv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_993\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_994\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fzvdai4edk5dwzv","anchor":"9109031"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-czgjanuqflyut2k\/","ref":"T332570#9109042"},[],{"anchor":"9109042"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_353\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1000\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1001\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_355\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-czgjanuqflyut2k\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1002\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1003\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-czgjanuqflyut2k","anchor":"9109042"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dya2sgn7gmyrzts","anchor":"9109043"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dvqyk2xt7fpmgsq\/","ref":"T332570#9113688"},[],{"anchor":"9113688"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_357\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1012\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1013\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_359\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dvqyk2xt7fpmgsq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1014\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1015\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dvqyk2xt7fpmgsq","anchor":"9113688"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2ndx2gj2uqyqvlf\/","ref":"T332570#9113690"},[],{"anchor":"9113690"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_361\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1021\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1022\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_363\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2ndx2gj2uqyqvlf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1023\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1024\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2ndx2gj2uqyqvlf","anchor":"9113690"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-herkju7wbf2k73k","anchor":"9113713"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jyqqf32oz4ijrqs\/","ref":"T332570#9113850"},[],{"anchor":"9113850"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_365\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1033\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1034\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_367\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jyqqf32oz4ijrqs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1035\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1036\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jyqqf32oz4ijrqs","anchor":"9113850"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tntne2obemjs6vs\/","ref":"T332570#9114023"},[],{"anchor":"9114023"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_369\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1042\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1043\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_371\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tntne2obemjs6vs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1044\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1045\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-tntne2obemjs6vs","anchor":"9114023"}],"javelin_behaviors":{"phui-hovercards":[],"phui-dropdown-menu":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/237d637a\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/1767ee66\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cda2b7e5\/core.pkg.css"]}