for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-ylxwwz7eyp5nsxppptd2\/?after=9319474&quoteTargetID=UQ0_1&quoteRef=T345811\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_242\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9319474\" id=\"9319474\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9319474\" data-sigil=\"has-tooltip\" data-meta=\"0_240\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 2:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 14:50:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_238\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/0NeQtIsBxE1_1c7smyH5\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_1\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-09T14:50:57Z] <fnegri@cloudcumin1001> END (FAIL) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=99) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_1\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_251\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9319489\" id=\"9319489\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_250\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9319489\" data-sigil=\"has-tooltip\" data-meta=\"0_249\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 2:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 14:52:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_247\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/BNeStIsBxE1_1c7sHyNv\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-09T14:52:37Z] <wm-bot2> fran@wmf3169 START - Cookbook wmcs.openstack.cloudvirt.drain on host 'cloudvirt1025.eqiad.wmnet' (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_260\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9319494\" id=\"9319494\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_259\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9319494\" data-sigil=\"has-tooltip\" data-meta=\"0_258\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 2:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 14:54:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_256\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/CXqTtIsBhuQtenzveQ1a\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-09T14:54:05Z] <wm-bot2> fran@wmf3169 admin END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) on host 'cloudvirt1025.eqiad.wmnet' (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_269\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9319571\" id=\"9319571\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_268\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9319571\" data-sigil=\"has-tooltip\" data-meta=\"0_267\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 3:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 15:12:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_265\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_266\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1025.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_278\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9319786\" id=\"9319786\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_277\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9319786\" data-sigil=\"has-tooltip\" data-meta=\"0_276\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 15:58:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_274\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_275\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1025.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1025 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091532_fnegri_1678960_cloudvirt1025.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_287\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320398\" id=\"9320398\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_286\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320398\" data-sigil=\"has-tooltip\" data-meta=\"0_285\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 5:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 17:50:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_283\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_284\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_47\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/pJ81tYsBGiVuUzOdZ71W\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_4\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-09T17:50:57Z] <fnegri@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.lib.ensure_canary (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_296\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320399\" id=\"9320399\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320399\" data-sigil=\"has-tooltip\" data-meta=\"0_294\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 5:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 17:51:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_292\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/9J81tYsBGiVuUzOdt72g\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_5\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-09T17:51:18Z] <fnegri@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.lib.ensure_canary (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_305\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320540\" id=\"9320540\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_304\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320540\" data-sigil=\"has-tooltip\" data-meta=\"0_303\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 6:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 18:24:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_301\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_302\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecloudvirt1025 had no VMs running (only the canary) because it's in the "maintenance" aggregate. Using the \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Portal:Cloud_VPS\/Admin\/Host_aggregates\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003efor loop described here\u003c\/a\u003e I listed the current aggregates for all cloudvirts: \u003cdiv class=\"paste-embed\"\u003e\u003cdiv class=\"paste-embed-head\"\u003e\u003ca href=\"\/P53231\"\u003eP53231 (An Untitled Masterwork)\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"paste-embed-body\" style=\"max-height: 27.6em;\"\u003e\u003cdiv class=\"phabricator-source-code-container\"\u003e\u003ctable class=\"phabricator-source-code-view remarkup-code PhabricatorMonospaced\" data-sigil=\"phabricator-source has-symbols\" data-meta=\"0_6\"\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e1\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003eroot@cloudcontrol1005:~# for i in $(sudo wmcs-openstack hypervisor list -f value -c ID); do sudo wmcs-openstack hypervisor show $i -c service_host -c aggregates ; done\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e2\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e3\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value           |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e4\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e5\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['maintenance'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e6\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1025   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e7\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e8\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e9\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e10\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e11\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e12\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1030 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e13\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e14\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e15\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e16\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e17\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e18\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1029 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e19\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e20\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e21\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value           |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e22\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e23\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['maintenance'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e24\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1026   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e25\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e26\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e27\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value           |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e28\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e29\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['maintenance'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e30\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1027   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e31\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e32\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e33\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e34\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e35\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['maintenance', 'ceph'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e36\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1028           |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e37\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e38\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e39\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e40\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e41\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e42\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1032 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e43\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e44\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e1+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e45\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e46\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e47\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e48\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1031 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e49\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e50\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e`1+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e51\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e52\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e53\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e54\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1033 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e55\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e56\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e57\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e58\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e59\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e60\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1034 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e61\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e62\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e63\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e64\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e65\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e66\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1036 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e67\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e68\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e69\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e70\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e71\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e72\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1037 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e73\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e74\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e75\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e76\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e77\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e78\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1035 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e79\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e80\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e81\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e82\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e83\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e84\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1039 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e85\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e86\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e87\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e88\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e89\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e90\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1040 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e91\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e92\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e93\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e94\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e95\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e96\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1044 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e97\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e98\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e99\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e100\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e101\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e102\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1042 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e103\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e104\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e105\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e106\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e107\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e108\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1045 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e109\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e110\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e111\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value           |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e112\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e113\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph', 'nfs'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e114\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1043   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e115\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e116\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e117\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e118\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e119\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e120\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1046 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e121\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e122\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e123\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e124\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e125\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e126\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1041 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e127\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e128\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e129\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e130\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e131\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e132\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1038 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e133\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e134\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e135\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e136\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e137\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e138\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1047 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e139\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e140\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e141\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e142\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e143\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['wdqs', 'standard'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e144\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt-wdqs1001   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e145\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e146\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e147\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e148\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e149\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['wdqs', 'standard'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e150\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt-wdqs1002   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e151\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e152\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e153\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e154\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e155\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['wdqs', 'standard'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e156\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt-wdqs1003   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e157\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+----------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e158\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e159\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e160\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e161\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e162\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1048 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e163\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e164\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e165\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e166\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e167\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e168\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1049 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e169\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e170\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e171\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e172\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e173\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e174\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1052 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e175\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e176\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e177\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e178\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e179\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e180\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1053 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e181\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e182\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e183\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e184\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e185\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e186\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1050 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e187\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e188\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e189\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value           |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e190\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e191\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['maintenance'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e192\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1051   |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e193\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+-----------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e194\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e195\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e196\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e197\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['spare']     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e198\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1057 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e199\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e200\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e201\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e202\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e203\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e204\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1056 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e205\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e206\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e207\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e208\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e209\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['spare']     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e210\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1061 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e211\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e212\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e213\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e214\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e215\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e216\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1058 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e217\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e218\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e219\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e220\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e221\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['spare']     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e222\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1060 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e223\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e224\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e225\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e226\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e227\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['spare']     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e228\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1059 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e229\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e230\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e231\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e232\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e233\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e234\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1055 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e235\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e236\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e237\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value         |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e238\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e239\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['ceph']      |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e240\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirt1054 |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e241\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e242\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e243\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e244\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e245\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['standard', 'localdisk'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e246\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirtlocal1001        |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e247\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e248\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e249\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e250\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e251\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['standard', 'localdisk'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e252\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirtlocal1002        |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e253\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e254\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e255\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| Field        | Value                     |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e256\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e257\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| aggregates   | ['standard', 'localdisk'] |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e258\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e| service_host | cloudvirtlocal1003        |\n\u003c\/td\u003e\u003c\/tr\u003e\u003ctr\u003e\u003cth class=\"phabricator-source-line\"\u003e\u003cspan\u003e259\u003c\/span\u003e\u003c\/th\u003e\u003ctd class=\"phabricator-source-code\"\u003e+--------------+---------------------------+\n\u003c\/td\u003e\u003c\/tr\u003e\u003c\/table\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_314\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320591\" id=\"9320591\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_313\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320591\" data-sigil=\"has-tooltip\" data-meta=\"0_312\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 6:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 18:35:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_310\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eUsing Cumin I checked the number of VMs currently running on all cloudvirts:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003efnegri@cloudcumin1001:~$ sudo cumin cloudvirt1* 'count=$(virsh list --uuid | grep [a-z0-9] | wc -l); if [[ $count -le 10 ]]; then echo "$count"; else echo "More than 10"; fi'\n43 hosts will be targeted:\ncloudvirt[1025-1067].eqiad.wmnet\nOK to proceed on 43 hosts? Enter the number of affected hosts to confirm or "q" to quit: 43\n===== NODE GROUP =====\n(6) cloudvirt[1062-1067].eqiad.wmnet\n----- OUTPUT of 'count=$(virsh li...ore than 10"; fi' -----\nbash: line 1: virsh: command not found\n0\n===== NODE GROUP =====\n(6) cloudvirt[1025-1027,1051,1059-1060].eqiad.wmnet\n----- OUTPUT of 'count=$(virsh li...ore than 10"; fi' -----\n1\n===== NODE GROUP =====\n(2) cloudvirt[1057,1061].eqiad.wmnet\n----- OUTPUT of 'count=$(virsh li...ore than 10"; fi' -----\n2\n===== NODE GROUP =====\n(29) cloudvirt[1028-1050,1052-1056,1058].eqiad.wmnet\n----- OUTPUT of 'count=$(virsh li...ore than 10"; fi' -----\nMore than 10\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eWe basically have 3 sets of "special" cloudvirts we need to treat differently during the reimage:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt[1062-1067].eqiad.wmnet\u003c\/tt\u003e are not in service yet (\u003ctt class=\"remarkup-monospaced\"\u003erole(insetup::wmcs)\u003c\/tt\u003e). We don't need to reimage them.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt[1025-1027,1051,1059-1060].eqiad.wmnet\u003c\/tt\u003e only have the canary VM running as they are either in \u003ctt class=\"remarkup-monospaced\"\u003eaggregate: maintenance\u003c\/tt\u003e or \u003ctt class=\"remarkup-monospaced\"\u003eaggregate: spare\u003c\/tt\u003e. We can reimage them all at once (after stopping the canary VM on them) and then use the \u003ctt class=\"remarkup-monospaced\"\u003ewmcs.openstack.cloudvirt.lib.ensure_canary\u003c\/tt\u003e to recreate a canary on them.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt[1057,1061].eqiad.wmnet\u003c\/tt\u003e are the trickiest: they have \u003ctt class=\"remarkup-monospaced\"\u003eaggregate: spare\u003c\/tt\u003e but they contain the canary \u003cstrong\u003eplus an extra VM\u003c\/strong\u003e. In cloudvirt1057 the extra VM is another canary (probably an error that can be ignored), while in cloudvirt1061 the extra VM is \u003ctt class=\"remarkup-monospaced\"\u003emwoffliner4\u003c\/tt\u003e, a VM that \u003ca href=\"\/p\/Andrew\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Andrew\u003c\/span\u003e\u003c\/a\u003e knows that is hard to move because of its size.\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_323\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320663\" id=\"9320663\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_322\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320663\" data-sigil=\"has-tooltip\" data-meta=\"0_321\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:02:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_319\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1026.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_332\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320678\" id=\"9320678\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_331\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320678\" data-sigil=\"has-tooltip\" data-meta=\"0_330\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:04:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_328\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_329\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1027.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_341\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320681\" id=\"9320681\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_340\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320681\" data-sigil=\"has-tooltip\" data-meta=\"0_339\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:04:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_337\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_338\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_59\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1057.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_350\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320685\" id=\"9320685\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_349\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320685\" data-sigil=\"has-tooltip\" data-meta=\"0_348\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:04:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_346\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_347\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1051.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_359\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320686\" id=\"9320686\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_358\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320686\" data-sigil=\"has-tooltip\" data-meta=\"0_357\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:04:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_355\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1059.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_368\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320687\" id=\"9320687\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_367\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320687\" data-sigil=\"has-tooltip\" data-meta=\"0_366\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:05:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_364\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1060.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_377\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320849\" id=\"9320849\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_376\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320849\" data-sigil=\"has-tooltip\" data-meta=\"0_375\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:43:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_373\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1027.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1027 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091919_fnegri_1780746_cloudvirt1027.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_386\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320862\" id=\"9320862\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_385\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320862\" data-sigil=\"has-tooltip\" data-meta=\"0_384\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:48:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_382\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1051.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1051 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eUnable to downtime the new host on Icinga\/Alertmanager, the sre.hosts.downtime cookbook returned 99\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091922_fnegri_1780872_cloudvirt1051.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCleared switch DHCP cache and MAC table for the host IP and MAC (row E\/F)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_395\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320867\" id=\"9320867\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_394\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320867\" data-sigil=\"has-tooltip\" data-meta=\"0_393\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:50:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_391\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1059.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1059 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091924_fnegri_1780965_cloudvirt1059.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eUnable to run puppet on config-master2001.codfw.wmnet,config-master1001.eqiad.wmnet to update configmaster.wikimedia.org with the new host SSH public key for wmf-update-known-hosts-production\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCleared switch DHCP cache and MAC table for the host IP and MAC (row E\/F)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_404\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320868\" id=\"9320868\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_403\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320868\" data-sigil=\"has-tooltip\" data-meta=\"0_402\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:50:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_400\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1026.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1026 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091922_fnegri_1780638_cloudvirt1026.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eUnable to run puppet on config-master2001.codfw.wmnet,config-master1001.eqiad.wmnet to update configmaster.wikimedia.org with the new host SSH public key for wmf-update-known-hosts-production\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_413\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320874\" id=\"9320874\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_412\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320874\" data-sigil=\"has-tooltip\" data-meta=\"0_411\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:52:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_409\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1057.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1057 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091927_fnegri_1782120_cloudvirt1057.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCleared switch DHCP cache and MAC table for the host IP and MAC (row E\/F)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_422\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320882\" id=\"9320882\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_421\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320882\" data-sigil=\"has-tooltip\" data-meta=\"0_420\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 7:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 19:54:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_418\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1060.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1060 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eUnable to downtime the new host on Icinga\/Alertmanager, the sre.hosts.downtime cookbook returned 99\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311091928_fnegri_1781452_cloudvirt1060.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCleared switch DHCP cache and MAC table for the host IP and MAC (row E\/F)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_431\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9320952\" id=\"9320952\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_430\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9320952\" data-sigil=\"has-tooltip\" data-meta=\"0_429\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2023, 8:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-09 20:15:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_427\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eReimaged so far: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt[1025-1027,1051,1057,1059-1060].eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_440\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9324304\" id=\"9324304\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_439\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9324304\" data-sigil=\"has-tooltip\" data-meta=\"0_438\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2023, 11:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-10 23:34:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_436\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1061.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_449\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v2iza5xhmlrh6nzci6b6\/PHID-FILE-jtmoofc7k3odr7cyyhyu\/profile-martian.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Andrew\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9324318\" id=\"9324318\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_448\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eAndrew\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9324318\" data-sigil=\"has-tooltip\" data-meta=\"0_447\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2023, 11:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-10 23:44:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_445\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_446\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003efyi \u003ca href=\"\/p\/fnegri\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fnegri\u003c\/span\u003e\u003c\/a\u003e, I predicted that draining cloudvirt1061 would be difficult due to mwoffliner4 but since my last attempt i did some tuning of the live-migration settings and it migrated just fine today. I'm now reimaging 1061 to Bookworm.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_458\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9324330\" id=\"9324330\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_457\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9324330\" data-sigil=\"has-tooltip\" data-meta=\"0_456\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2023, 12:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-11 00:14:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_454\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1061.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1061 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311102351_andrew_2548168_cloudvirt1061.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCleared switch DHCP cache and MAC table for the host IP and MAC (row E\/F)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_467\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9324355\" id=\"9324355\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_466\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9324355\" data-sigil=\"has-tooltip\" data-meta=\"0_465\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2023, 1:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-11 01:03:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_463\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_464\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1058.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_476\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9324362\" id=\"9324362\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_475\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9324362\" data-sigil=\"has-tooltip\" data-meta=\"0_474\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2023, 1:47 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-11 01:47:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_472\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_473\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1058.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1058 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311110120_andrew_2587193_cloudvirt1058.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCleared switch DHCP cache and MAC table for the host IP and MAC (row E\/F)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_485\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9326871\" id=\"9326871\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_484\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9326871\" data-sigil=\"has-tooltip\" data-meta=\"0_483\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 3:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 15:19:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_481\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_482\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/TY5DyYsBhuQtenzv0bWP\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_6\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T15:19:05Z] <wm-bot2> fran@wmf3169 START - Cookbook wmcs.openstack.cloudvirt.drain on host 'cloudvirt1031.eqiad.wmnet' (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_494\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9326949\" id=\"9326949\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_493\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9326949\" data-sigil=\"has-tooltip\" data-meta=\"0_492\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 3:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 15:37:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_490\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_491\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/9upVyYsBxE1_1c7sDAbz\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_7\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T15:37:55Z] <wm-bot2> fran@wmf3169 admin END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) on host 'cloudvirt1031.eqiad.wmnet' (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_503\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327039\" id=\"9327039\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_502\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327039\" data-sigil=\"has-tooltip\" data-meta=\"0_501\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 3:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 15:55:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_499\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_500\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by fnegri@cumin1001 for host cloudvirt1031.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_512\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327286\" id=\"9327286\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_511\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327286\" data-sigil=\"has-tooltip\" data-meta=\"0_510\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 4:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 16:40:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_508\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_509\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by fnegri@cumin1001 for host cloudvirt1031.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1031 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311131612_fnegri_4167178_cloudvirt1031.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_521\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327341\" id=\"9327341\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_520\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327341\" data-sigil=\"has-tooltip\" data-meta=\"0_519\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 4:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 16:56:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_517\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_518\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/2OqdyYsBxE1_1c7sVEjz\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_8\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T16:56:52Z] <wm-bot2> fran@wmf3169 START - Cookbook wmcs.openstack.cloudvirt.unset_maintenance (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_530\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327345\" id=\"9327345\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_529\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327345\" data-sigil=\"has-tooltip\" data-meta=\"0_528\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 4:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 16:57:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_526\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_527\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_101\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/8eqdyYsBxE1_1c7sk0h9\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_9\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T16:57:09Z] <wm-bot2> fran@wmf3169 END (FAIL) - Cookbook wmcs.openstack.cloudvirt.unset_maintenance (exit_code=99) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_539\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327384\" id=\"9327384\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_538\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327384\" data-sigil=\"has-tooltip\" data-meta=\"0_537\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 5:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 17:09:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_535\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/H-qoyYsBxE1_1c7srVL6\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_10\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T17:09:16Z] <wm-bot2> fran@wmf3169 START - Cookbook wmcs.openstack.cloudvirt.unset_maintenance (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_548\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327386\" id=\"9327386\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_547\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327386\" data-sigil=\"has-tooltip\" data-meta=\"0_546\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 5:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 17:09:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_544\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_545\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/IuqoyYsBxE1_1c7s9lJP\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_11\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T17:09:35Z] <wm-bot2> fran@wmf3169 END (PASS) - Cookbook wmcs.openstack.cloudvirt.unset_maintenance (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_557\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327715\" id=\"9327715\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_556\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327715\" data-sigil=\"has-tooltip\" data-meta=\"0_555\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:00:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_553\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_554\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_566\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327716\" id=\"9327716\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_565\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327716\" data-sigil=\"has-tooltip\" data-meta=\"0_564\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:01:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_562\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_563\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_109\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/7LQPyosBGiVuUzOdJX_R\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_12\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T19:01:11Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_575\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327770\" id=\"9327770\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_574\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327770\" data-sigil=\"has-tooltip\" data-meta=\"0_573\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:15:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_571\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_572\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_111\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1032 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_584\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327774\" id=\"9327774\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_583\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327774\" data-sigil=\"has-tooltip\" data-meta=\"0_582\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:17:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_580\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_581\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_113\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/prQdyosBGiVuUzOd94ao\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_13\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T19:17:23Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_593\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327778\" id=\"9327778\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_592\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327778\" data-sigil=\"has-tooltip\" data-meta=\"0_591\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:19:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_589\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_590\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_602\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327780\" id=\"9327780\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_601\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327780\" data-sigil=\"has-tooltip\" data-meta=\"0_600\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:20:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_598\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_599\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_611\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327805\" id=\"9327805\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_610\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327805\" data-sigil=\"has-tooltip\" data-meta=\"0_609\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:35:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_607\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_608\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_119\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1033 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_620\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327810\" id=\"9327810\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_619\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_120\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327810\" data-sigil=\"has-tooltip\" data-meta=\"0_618\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:37:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_616\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_617\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1032 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_629\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327811\" id=\"9327811\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_628\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327811\" data-sigil=\"has-tooltip\" data-meta=\"0_627\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:38:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_625\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_626\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_638\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327812\" id=\"9327812\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_637\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327812\" data-sigil=\"has-tooltip\" data-meta=\"0_636\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:38:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_634\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_635\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_647\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327825\" id=\"9327825\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_646\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327825\" data-sigil=\"has-tooltip\" data-meta=\"0_645\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:53:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_643\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_644\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_127\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1033 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_656\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327839\" id=\"9327839\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_655\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_128\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327839\" data-sigil=\"has-tooltip\" data-meta=\"0_654\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:56:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_652\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_653\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_129\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_665\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327851\" id=\"9327851\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_664\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327851\" data-sigil=\"has-tooltip\" data-meta=\"0_663\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 7:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 19:59:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_661\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_662\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1032 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_674\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327853\" id=\"9327853\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_673\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327853\" data-sigil=\"has-tooltip\" data-meta=\"0_672\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:00:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_670\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_671\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_683\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327868\" id=\"9327868\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_682\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327868\" data-sigil=\"has-tooltip\" data-meta=\"0_681\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:09:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_679\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_680\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1033 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_692\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327881\" id=\"9327881\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_691\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327881\" data-sigil=\"has-tooltip\" data-meta=\"0_690\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:18:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_688\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_689\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_701\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327904\" id=\"9327904\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_700\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327904\" data-sigil=\"has-tooltip\" data-meta=\"0_699\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:35:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_697\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_698\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by andrew@cumin1001 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1025.eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1025.eqiad.wmnet (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_710\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327916\" id=\"9327916\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_709\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327916\" data-sigil=\"has-tooltip\" data-meta=\"0_708\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:41:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_706\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_707\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1032.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1032 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132017_andrew_80224_cloudvirt1032.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_719\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327919\" id=\"9327919\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_718\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_142\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327919\" data-sigil=\"has-tooltip\" data-meta=\"0_717\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:43:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_715\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_716\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1034.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_728\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327920\" id=\"9327920\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_727\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_144\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327920\" data-sigil=\"has-tooltip\" data-meta=\"0_726\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:43:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_724\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_725\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_145\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by andrew@cumin1001 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1026.eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1026.eqiad.wmnet (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_737\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327957\" id=\"9327957\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_736\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_146\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327957\" data-sigil=\"has-tooltip\" data-meta=\"0_735\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:52:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_733\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_734\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_147\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by andrew@cumin1001 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1027.eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1027.eqiad.wmnet (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_746\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327983\" id=\"9327983\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_745\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_148\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327983\" data-sigil=\"has-tooltip\" data-meta=\"0_744\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 8:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 20:59:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_742\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_743\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_149\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1033.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1033 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132036_andrew_88814_cloudvirt1033.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_755\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327984\" id=\"9327984\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_754\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_150\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327984\" data-sigil=\"has-tooltip\" data-meta=\"0_753\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:00:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_751\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_752\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_151\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/KZB8yosBhuQtenzvIjqw\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_14\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T21:00:14Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_764\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9327990\" id=\"9327990\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_763\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_152\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9327990\" data-sigil=\"has-tooltip\" data-meta=\"0_762\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:01:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_760\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_761\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by andrew@cumin1001 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1028.eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1028.eqiad.wmnet (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_773\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328047\" id=\"9328047\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_772\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328047\" data-sigil=\"has-tooltip\" data-meta=\"0_771\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:10:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_769\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_770\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_155\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by andrew@cumin1001 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1029.eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1029.eqiad.wmnet (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_782\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328076\" id=\"9328076\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_781\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_156\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328076\" data-sigil=\"has-tooltip\" data-meta=\"0_780\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:18:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_778\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_779\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_157\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1035.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_791\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328077\" id=\"9328077\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_790\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_158\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328077\" data-sigil=\"has-tooltip\" data-meta=\"0_789\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:18:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_787\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_788\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_159\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by andrew@cumin1001 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1030.eqiad.wmnet\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1030.eqiad.wmnet (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_800\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328084\" id=\"9328084\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_799\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_160\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328084\" data-sigil=\"has-tooltip\" data-meta=\"0_798\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:21:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_796\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_797\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/q-uPyosBxE1_1c7saXXe\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_15\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T21:21:18Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_809\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328130\" id=\"9328130\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_808\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_162\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328130\" data-sigil=\"has-tooltip\" data-meta=\"0_807\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:25:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_805\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_806\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_163\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1034.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1034 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132102_andrew_101620_cloudvirt1034.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_818\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328175\" id=\"9328175\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_817\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_164\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328175\" data-sigil=\"has-tooltip\" data-meta=\"0_816\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:30:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_814\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_815\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_165\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/P5CXyosBhuQtenzvk1O6\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_16\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T21:30:12Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_827\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328179\" id=\"9328179\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_826\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_166\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328179\" data-sigil=\"has-tooltip\" data-meta=\"0_825\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:30:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_823\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_824\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_167\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1036.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_836\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328246\" id=\"9328246\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_835\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_168\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328246\" data-sigil=\"has-tooltip\" data-meta=\"0_834\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 9:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 21:46:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_832\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_833\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/9ZCmyosBhuQtenzvzmyH\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_17\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T21:46:50Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_845\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328293\" id=\"9328293\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_844\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_170\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328293\" data-sigil=\"has-tooltip\" data-meta=\"0_843\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:05:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_841\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_842\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1035.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1035 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132137_andrew_120932_cloudvirt1035.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_854\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328296\" id=\"9328296\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_853\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_172\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328296\" data-sigil=\"has-tooltip\" data-meta=\"0_852\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:08:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_850\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_851\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_173\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/Q-u6yosBxE1_1c7s87H4\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_18\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T22:08:51Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_863\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328297\" id=\"9328297\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_862\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_174\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328297\" data-sigil=\"has-tooltip\" data-meta=\"0_861\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:09:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_859\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_860\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_175\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/FpC7yosBhuQtenzvj5EB\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_19\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T22:09:31Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_872\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328301\" id=\"9328301\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_871\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_176\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328301\" data-sigil=\"has-tooltip\" data-meta=\"0_870\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:10:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_868\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_869\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1037.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_881\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328307\" id=\"9328307\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_880\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_178\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328307\" data-sigil=\"has-tooltip\" data-meta=\"0_879\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:17:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_877\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_878\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1036.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1036 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132148_andrew_127394_cloudvirt1036.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_890\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328335\" id=\"9328335\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_889\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328335\" data-sigil=\"has-tooltip\" data-meta=\"0_888\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:23:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_886\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_887\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1038.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_899\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328351\" id=\"9328351\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_898\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328351\" data-sigil=\"has-tooltip\" data-meta=\"0_897\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:24:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_895\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_896\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_183\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/dZDJyosBhuQtenzvb5sP\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_20\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T22:24:40Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_908\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328354\" id=\"9328354\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_907\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_184\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328354\" data-sigil=\"has-tooltip\" data-meta=\"0_906\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:25:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_904\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_905\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_185\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/O5DJyosBhuQtenzv_ZxB\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_21\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-13T22:25:17Z] <andrew@cloudcumin1001> END (FAIL) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=99) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_917\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328452\" id=\"9328452\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_916\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_186\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328452\" data-sigil=\"has-tooltip\" data-meta=\"0_915\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 10:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 22:59:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_913\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_914\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_187\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1037.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1037 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132232_andrew_146966_cloudvirt1037.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_926\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328476\" id=\"9328476\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_925\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_188\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328476\" data-sigil=\"has-tooltip\" data-meta=\"0_924\"\u003e\u003cspan class=\"screen-only\"\u003eNov 13 2023, 11:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-13 23:10:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_922\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_923\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1038.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1038 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311132242_andrew_152469_cloudvirt1038.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_935\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9328946\" id=\"9328946\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_934\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_190\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9328946\" data-sigil=\"has-tooltip\" data-meta=\"0_933\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:00:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_931\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_932\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_191\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/jOyOy4sBxE1_1c7s8l9y\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_22\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T02:00:24Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_944\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329027\" id=\"9329027\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_943\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_192\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329027\" data-sigil=\"has-tooltip\" data-meta=\"0_942\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:23:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_940\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_941\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_193\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/Yraky4sBGiVuUzOdCh-i\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_23\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T02:23:26Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_953\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329045\" id=\"9329045\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_952\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329045\" data-sigil=\"has-tooltip\" data-meta=\"0_951\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:26:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_949\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_950\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_195\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1039.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_962\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329077\" id=\"9329077\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_961\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_196\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329077\" data-sigil=\"has-tooltip\" data-meta=\"0_960\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:34 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:34:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_958\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_959\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_197\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/ubauy4sBGiVuUzOdMSIv\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_24\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T02:34:32Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_971\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329104\" id=\"9329104\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_970\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_198\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329104\" data-sigil=\"has-tooltip\" data-meta=\"0_969\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:52 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:52:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_967\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_968\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_199\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/8-y-y4sBxE1_1c7sP4Nn\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_25\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T02:52:03Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_980\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329143\" id=\"9329143\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_979\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_200\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329143\" data-sigil=\"has-tooltip\" data-meta=\"0_978\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:59:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_976\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_977\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_201\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/WbbEy4sBGiVuUzOdnzWi\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_26\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T02:59:01Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_989\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329144\" id=\"9329144\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_988\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_202\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329144\" data-sigil=\"has-tooltip\" data-meta=\"0_987\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:59:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_985\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_986\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_203\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/pbbFy4sBGiVuUzOdUTWG\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_27\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T02:59:48Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_998\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329145\" id=\"9329145\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_997\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_204\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329145\" data-sigil=\"has-tooltip\" data-meta=\"0_996\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 2:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 02:59:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_994\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_995\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_205\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1040.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1007\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329157\" id=\"9329157\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1006\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_206\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329157\" data-sigil=\"has-tooltip\" data-meta=\"0_1005\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:08:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1003\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1004\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_207\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/P-zNy4sBxE1_1c7sA437\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_28\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T03:08:12Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1016\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329158\" id=\"9329158\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1015\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_208\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329158\" data-sigil=\"has-tooltip\" data-meta=\"0_1014\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:08:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1012\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1013\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_209\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1039.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1039 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311140246_andrew_261589_cloudvirt1039.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1025\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329160\" id=\"9329160\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1024\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_210\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329160\" data-sigil=\"has-tooltip\" data-meta=\"0_1023\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:09:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1021\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1022\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_211\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1041.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1034\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329177\" id=\"9329177\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1033\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_212\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329177\" data-sigil=\"has-tooltip\" data-meta=\"0_1032\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:26:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1030\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1031\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_213\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/l5Hdy4sBhuQtenzvWYlR\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_29\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T03:26:02Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1043\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329186\" id=\"9329186\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1042\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_214\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329186\" data-sigil=\"has-tooltip\" data-meta=\"0_1041\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:33:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1039\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1040\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_215\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1042.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1052\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329192\" id=\"9329192\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1051\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_216\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329192\" data-sigil=\"has-tooltip\" data-meta=\"0_1050\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:43 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:43:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1048\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1049\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_217\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1040.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1040 (\u003cstrong\u003eWARN\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311140316_andrew_276937_cloudvirt1040.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eIcinga status is not optimal, downtime not removed\u003c\/em\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1061\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329194\" id=\"9329194\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1060\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_218\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329194\" data-sigil=\"has-tooltip\" data-meta=\"0_1059\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:46:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1057\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1058\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_219\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1042.eqiad.wmnet with OS bookworm executed with errors:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1042 (\u003cstrong\u003eFAIL\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eThe reimage failed, see the cookbook logs for the details\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1070\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329197\" id=\"9329197\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1069\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_220\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329197\" data-sigil=\"has-tooltip\" data-meta=\"0_1068\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:48:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1066\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1067\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_221\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1042.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1079\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329198\" id=\"9329198\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1078\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_222\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329198\" data-sigil=\"has-tooltip\" data-meta=\"0_1077\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:48:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1075\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1076\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_223\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/ebbxy4sBGiVuUzOd41Gh\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_30\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T03:48:29Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1088\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329200\" id=\"9329200\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1087\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_224\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329200\" data-sigil=\"has-tooltip\" data-meta=\"0_1086\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:49:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1084\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1085\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_225\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by andrew@cumin1001 for host cloudvirt1041.eqiad.wmnet with OS bookworm completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudvirt1041 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled Puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bookworm OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved previous downtime on Alertmanager (old OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202311140325_andrew_280592_cloudvirt1041.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1097\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329201\" id=\"9329201\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1096\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_226\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329201\" data-sigil=\"has-tooltip\" data-meta=\"0_1095\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:49:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1093\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1094\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_227\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/QJHyy4sBhuQtenzvgJQH\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_31\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T03:49:09Z] <andrew@cloudcumin1001> END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1106\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329203\" id=\"9329203\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1105\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_228\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329203\" data-sigil=\"has-tooltip\" data-meta=\"0_1104\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:49:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1102\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1103\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_229\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by andrew@cumin1001 for host cloudvirt1043.eqiad.wmnet with OS bookworm\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1115\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329204\" id=\"9329204\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_230\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329204\" data-sigil=\"has-tooltip\" data-meta=\"0_1113\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 3:50 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 03:50:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1111\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1112\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_231\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/puzzy4sBxE1_1c7si5-z\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_32\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T03:50:17Z] <andrew@cloudcumin1001> START - Cookbook wmcs.openstack.cloudvirt.drain (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1124\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9329228\" id=\"9329228\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1123\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_232\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9329228\" data-sigil=\"has-tooltip\" data-meta=\"0_1122\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2023, 4:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-11-14 04:11:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1120\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1121\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_233\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/V-wGzIsBxE1_1c7s8K1M\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_33\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2023-11-14T04:11:27Z] <andrew@cloudcumin1001> END (FAIL) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=99) (\u003ca href=\"\/T345811\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT345811\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"uri":"","symbols":null},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu","contextPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im","contextPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ylxwwz7eyp5nsxppptd2"}},{"phid":"PHID-XACT-TASK-2b4ptx6rxajtlox"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-m5cnuzvgyqdvrd4"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-3qo4wzd5oslws5z"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-6nvat7rfbss44sv"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2fxn2ifrqhvvb3u"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-o7azgvhu7mb2raf"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-szwodlwbxcnvbs5"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-2j7bajx3iksvcvg"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-zcrjqdakifnlkmo"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kiumcugfgycuggs"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2ljvfwp7dv5ovmq"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-caflragkoyakkfn"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2wgnwts34cwtzzy"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-6fyqdciiwqgmywj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-qpollsaxy4uzpns"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-7xa5i6o7kxzraff"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-g76gz4v27vpy3wa"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-43vtamm3ahaj6un"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fpm4ikorcdy7rmx"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-bphvrygttdnnyhq"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-gyquu7la4kwaom4"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-xyn3xkgcd7pvybs"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-n6mawte3rhifnh6"},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu"}},{"phid":"PHID-XACT-TASK-5ckm2rkra63kiye"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3od6bcoseqjkqwj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-dvudhe6cppp4hri"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-dv67mkk2mdpvyoy"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-7nul22lbgnbr6tq"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-fgqntvs64ankxoo"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-b242qlcwur4hr4x"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-xbkoczrgyyeq6ib"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-gkwswg4cgjpd2gl"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-q75fnnzaj2uxar3"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-k7xzihxlbmsffca"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-upn6ny75zaesnkp"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kocu2n2er5dv6py"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-rzdrymj4toknhcl"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-bz7pkzb36tpqn2w"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-ayx4wj46spboz4e"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-7bhtqlojb34r7qf"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kpi2gf3vyglpceg"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-rbrncyh7cckhkiq"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-qkhzq6twxwsnhxy"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-jncnysjsxcx53qk"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-duo6jxfpbtmfa5n"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-tzfs2vgwmn75e3p"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-w2kngh7kybwkxnt"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-wf5wautagomjx2t"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-mjmd3teg37lgo3i"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-obnynj7h2xykbiz"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-pxlikoz44felfqz"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kfg2lm5z6mpvunk"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-jd7oahf3nxzve2r"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-llx6zxyxt3yy25w"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-usc54jwu7t2hnoa"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-4asbjgpobnapsag"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ub7ro3bd77y6ze5"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-etqld2dk3phqvxl"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-e3v4xlefzopc4oy"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ojnk3b5comhumj4"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fsxyivz7lwuexa6"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fhj5tlgxf2snszd"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-fhsg4c2w4r3l2ib"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3o2vlihb4bgbjdg"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-7pv6euzdn42y7tk"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3l72z2iuln3z7g7"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-eu6fpxxdxr7mcqa"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-as2vjm2udhji6cx"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-5jh4kwvc4si7v7c"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-wscgiq6cooi5lfa"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-ofwka6eibdp7c74"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-7ku44onlnsp453u"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-6m336cwhhi5qkv5"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-orqpewtplpvzdvs"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-34pjmpaz242vgey"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-uyqezligowagvft"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-mnfo5k7unmkazbh"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-dbybxohqf7skket"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-3he2lj6pprcrpwm"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-bpejlmeookin4xl"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-i7foqmt7fj32rmp"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-znawqneokj2mrjx"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-ak2zyn7u6ez7btk"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-z6o44xdgmwayqxr"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-egegman4ozic2iu"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-x4ktdmn7abmcbrw"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-zuqxnsgnzpqfla2"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-2tfjtchji6x2gvy"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-kbcvl5radm4hwsd"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-aayl2arp6qhs2ol"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-mjshsmks6ws4aud"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-wph2wulbtwesuqo"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-7lgbfh3aven2ntq"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-xo2qldz4vn354y4"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kgrtnrzyqpedk7p"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-zejup2ht3i76xij"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-enjs32bm646briv"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-rnggxtmym65jntx"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-rq5usdeo6x4keqc"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2b4ptx6rxajtlox\/","ref":"T345811#9319474"},[],{"anchor":"9319474"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_234\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_235\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2b4ptx6rxajtlox\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_236\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2b4ptx6rxajtlox","anchor":"9319474"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-m5cnuzvgyqdvrd4\/","ref":"T345811#9319489"},[],{"anchor":"9319489"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_243\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_244\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-m5cnuzvgyqdvrd4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_245\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-m5cnuzvgyqdvrd4","anchor":"9319489"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3qo4wzd5oslws5z\/","ref":"T345811#9319494"},[],{"anchor":"9319494"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3qo4wzd5oslws5z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_254\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3qo4wzd5oslws5z","anchor":"9319494"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6nvat7rfbss44sv\/","ref":"T345811#9319571"},[],{"anchor":"9319571"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_261\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6nvat7rfbss44sv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_263\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6nvat7rfbss44sv","anchor":"9319571"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2fxn2ifrqhvvb3u\/","ref":"T345811#9319786"},[],{"anchor":"9319786"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_270\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2fxn2ifrqhvvb3u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_272\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2fxn2ifrqhvvb3u","anchor":"9319786"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-o7azgvhu7mb2raf\/","ref":"T345811#9320398"},[],{"anchor":"9320398"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_279\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-o7azgvhu7mb2raf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_281\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-o7azgvhu7mb2raf","anchor":"9320398"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-szwodlwbxcnvbs5\/","ref":"T345811#9320399"},[],{"anchor":"9320399"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_288\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-szwodlwbxcnvbs5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_290\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-szwodlwbxcnvbs5","anchor":"9320399"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2j7bajx3iksvcvg\/","ref":"T345811#9320540"},[],{"anchor":"9320540"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_297\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2j7bajx3iksvcvg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_299\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2j7bajx3iksvcvg","anchor":"9320540"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zcrjqdakifnlkmo\/","ref":"T345811#9320591"},[],{"anchor":"9320591"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_306\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zcrjqdakifnlkmo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_308\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_309\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-zcrjqdakifnlkmo","anchor":"9320591"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kiumcugfgycuggs\/","ref":"T345811#9320663"},[],{"anchor":"9320663"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_315\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kiumcugfgycuggs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_317\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_318\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kiumcugfgycuggs","anchor":"9320663"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2ljvfwp7dv5ovmq\/","ref":"T345811#9320678"},[],{"anchor":"9320678"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_324\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2ljvfwp7dv5ovmq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_326\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_327\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2ljvfwp7dv5ovmq","anchor":"9320678"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-caflragkoyakkfn\/","ref":"T345811#9320681"},[],{"anchor":"9320681"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_333\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-caflragkoyakkfn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_335\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-caflragkoyakkfn","anchor":"9320681"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2wgnwts34cwtzzy\/","ref":"T345811#9320685"},[],{"anchor":"9320685"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_342\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2wgnwts34cwtzzy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_344\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_345\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2wgnwts34cwtzzy","anchor":"9320685"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6fyqdciiwqgmywj\/","ref":"T345811#9320686"},[],{"anchor":"9320686"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_351\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6fyqdciiwqgmywj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_353\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_354\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6fyqdciiwqgmywj","anchor":"9320686"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qpollsaxy4uzpns\/","ref":"T345811#9320687"},[],{"anchor":"9320687"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_360\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_361\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qpollsaxy4uzpns\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_362\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_363\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-qpollsaxy4uzpns","anchor":"9320687"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7xa5i6o7kxzraff\/","ref":"T345811#9320849"},[],{"anchor":"9320849"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_369\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_370\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7xa5i6o7kxzraff\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_371\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_372\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7xa5i6o7kxzraff","anchor":"9320849"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-g76gz4v27vpy3wa\/","ref":"T345811#9320862"},[],{"anchor":"9320862"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_378\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-g76gz4v27vpy3wa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_380\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-g76gz4v27vpy3wa","anchor":"9320862"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-43vtamm3ahaj6un\/","ref":"T345811#9320867"},[],{"anchor":"9320867"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_387\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-43vtamm3ahaj6un\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_389\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-43vtamm3ahaj6un","anchor":"9320867"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fpm4ikorcdy7rmx\/","ref":"T345811#9320868"},[],{"anchor":"9320868"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_396\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_397\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fpm4ikorcdy7rmx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_398\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fpm4ikorcdy7rmx","anchor":"9320868"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bphvrygttdnnyhq\/","ref":"T345811#9320874"},[],{"anchor":"9320874"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_405\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_406\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bphvrygttdnnyhq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_407\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_408\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bphvrygttdnnyhq","anchor":"9320874"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gyquu7la4kwaom4\/","ref":"T345811#9320882"},[],{"anchor":"9320882"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_414\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_415\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gyquu7la4kwaom4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_416\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_417\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gyquu7la4kwaom4","anchor":"9320882"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xyn3xkgcd7pvybs\/","ref":"T345811#9320952"},[],{"anchor":"9320952"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_423\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xyn3xkgcd7pvybs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_425\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xyn3xkgcd7pvybs","anchor":"9320952"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-n6mawte3rhifnh6\/","ref":"T345811#9324304"},[],{"anchor":"9324304"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_432\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-n6mawte3rhifnh6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_434\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-n6mawte3rhifnh6","anchor":"9324304"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5ckm2rkra63kiye\/","ref":"T345811#9324318"},[],{"anchor":"9324318"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_441\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5ckm2rkra63kiye\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_443\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5ckm2rkra63kiye","anchor":"9324318"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3od6bcoseqjkqwj\/","ref":"T345811#9324330"},[],{"anchor":"9324330"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_450\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3od6bcoseqjkqwj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_452\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3od6bcoseqjkqwj","anchor":"9324330"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dvudhe6cppp4hri\/","ref":"T345811#9324355"},[],{"anchor":"9324355"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_459\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_460\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dvudhe6cppp4hri\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_461\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_462\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dvudhe6cppp4hri","anchor":"9324355"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dv67mkk2mdpvyoy\/","ref":"T345811#9324362"},[],{"anchor":"9324362"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_468\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_469\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dv67mkk2mdpvyoy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_470\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_471\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dv67mkk2mdpvyoy","anchor":"9324362"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7nul22lbgnbr6tq\/","ref":"T345811#9326871"},[],{"anchor":"9326871"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_477\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_478\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7nul22lbgnbr6tq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_479\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_480\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7nul22lbgnbr6tq","anchor":"9326871"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fgqntvs64ankxoo\/","ref":"T345811#9326949"},[],{"anchor":"9326949"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_486\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_487\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fgqntvs64ankxoo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_488\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_489\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fgqntvs64ankxoo","anchor":"9326949"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-b242qlcwur4hr4x\/","ref":"T345811#9327039"},[],{"anchor":"9327039"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_495\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_496\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-b242qlcwur4hr4x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_497\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_498\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-b242qlcwur4hr4x","anchor":"9327039"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xbkoczrgyyeq6ib\/","ref":"T345811#9327286"},[],{"anchor":"9327286"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_504\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_505\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xbkoczrgyyeq6ib\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_506\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_507\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xbkoczrgyyeq6ib","anchor":"9327286"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gkwswg4cgjpd2gl\/","ref":"T345811#9327341"},[],{"anchor":"9327341"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_513\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_514\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gkwswg4cgjpd2gl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_515\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_516\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gkwswg4cgjpd2gl","anchor":"9327341"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-q75fnnzaj2uxar3\/","ref":"T345811#9327345"},[],{"anchor":"9327345"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_522\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_523\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-q75fnnzaj2uxar3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_524\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_525\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-q75fnnzaj2uxar3","anchor":"9327345"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-k7xzihxlbmsffca\/","ref":"T345811#9327384"},[],{"anchor":"9327384"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_531\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_532\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-k7xzihxlbmsffca\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_533\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_534\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-k7xzihxlbmsffca","anchor":"9327384"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-upn6ny75zaesnkp\/","ref":"T345811#9327386"},[],{"anchor":"9327386"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_540\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_541\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-upn6ny75zaesnkp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_542\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_543\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-upn6ny75zaesnkp","anchor":"9327386"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kocu2n2er5dv6py\/","ref":"T345811#9327715"},[],{"anchor":"9327715"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_549\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_550\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kocu2n2er5dv6py\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_551\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_552\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kocu2n2er5dv6py","anchor":"9327715"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rzdrymj4toknhcl\/","ref":"T345811#9327716"},[],{"anchor":"9327716"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_558\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_559\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rzdrymj4toknhcl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_560\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_561\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rzdrymj4toknhcl","anchor":"9327716"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bz7pkzb36tpqn2w\/","ref":"T345811#9327770"},[],{"anchor":"9327770"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_567\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_568\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bz7pkzb36tpqn2w\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_569\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_570\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bz7pkzb36tpqn2w","anchor":"9327770"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ayx4wj46spboz4e\/","ref":"T345811#9327774"},[],{"anchor":"9327774"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_576\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_577\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ayx4wj46spboz4e\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_578\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_579\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ayx4wj46spboz4e","anchor":"9327774"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7bhtqlojb34r7qf\/","ref":"T345811#9327778"},[],{"anchor":"9327778"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_585\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_586\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7bhtqlojb34r7qf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_587\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_588\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7bhtqlojb34r7qf","anchor":"9327778"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kpi2gf3vyglpceg\/","ref":"T345811#9327780"},[],{"anchor":"9327780"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_594\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_595\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kpi2gf3vyglpceg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_596\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_597\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kpi2gf3vyglpceg","anchor":"9327780"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rbrncyh7cckhkiq\/","ref":"T345811#9327805"},[],{"anchor":"9327805"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_603\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_604\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rbrncyh7cckhkiq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_605\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_606\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rbrncyh7cckhkiq","anchor":"9327805"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qkhzq6twxwsnhxy\/","ref":"T345811#9327810"},[],{"anchor":"9327810"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_612\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_613\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qkhzq6twxwsnhxy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_614\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_615\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-qkhzq6twxwsnhxy","anchor":"9327810"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jncnysjsxcx53qk\/","ref":"T345811#9327811"},[],{"anchor":"9327811"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_621\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_622\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jncnysjsxcx53qk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_623\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_624\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jncnysjsxcx53qk","anchor":"9327811"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-duo6jxfpbtmfa5n\/","ref":"T345811#9327812"},[],{"anchor":"9327812"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_630\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_631\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-duo6jxfpbtmfa5n\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_632\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_633\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-duo6jxfpbtmfa5n","anchor":"9327812"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tzfs2vgwmn75e3p\/","ref":"T345811#9327825"},[],{"anchor":"9327825"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_639\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_640\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tzfs2vgwmn75e3p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_641\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_642\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-tzfs2vgwmn75e3p","anchor":"9327825"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-w2kngh7kybwkxnt\/","ref":"T345811#9327839"},[],{"anchor":"9327839"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_648\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_649\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-w2kngh7kybwkxnt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_650\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_651\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-w2kngh7kybwkxnt","anchor":"9327839"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wf5wautagomjx2t\/","ref":"T345811#9327851"},[],{"anchor":"9327851"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_657\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_658\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wf5wautagomjx2t\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_659\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_660\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wf5wautagomjx2t","anchor":"9327851"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mjmd3teg37lgo3i\/","ref":"T345811#9327853"},[],{"anchor":"9327853"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_666\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_667\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mjmd3teg37lgo3i\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_668\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_669\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mjmd3teg37lgo3i","anchor":"9327853"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-obnynj7h2xykbiz\/","ref":"T345811#9327868"},[],{"anchor":"9327868"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_675\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_676\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-obnynj7h2xykbiz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_677\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-obnynj7h2xykbiz","anchor":"9327868"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pxlikoz44felfqz\/","ref":"T345811#9327881"},[],{"anchor":"9327881"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_684\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_685\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pxlikoz44felfqz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_686\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_687\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pxlikoz44felfqz","anchor":"9327881"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kfg2lm5z6mpvunk\/","ref":"T345811#9327904"},[],{"anchor":"9327904"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_693\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_694\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kfg2lm5z6mpvunk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_695\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_696\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kfg2lm5z6mpvunk","anchor":"9327904"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jd7oahf3nxzve2r\/","ref":"T345811#9327916"},[],{"anchor":"9327916"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_702\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_703\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jd7oahf3nxzve2r\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_704\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_705\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jd7oahf3nxzve2r","anchor":"9327916"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-llx6zxyxt3yy25w\/","ref":"T345811#9327919"},[],{"anchor":"9327919"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_711\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_712\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_215\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-llx6zxyxt3yy25w\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_713\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-llx6zxyxt3yy25w","anchor":"9327919"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-usc54jwu7t2hnoa\/","ref":"T345811#9327920"},[],{"anchor":"9327920"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_217\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_720\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_721\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_219\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-usc54jwu7t2hnoa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_722\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_723\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-usc54jwu7t2hnoa","anchor":"9327920"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4asbjgpobnapsag\/","ref":"T345811#9327957"},[],{"anchor":"9327957"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_221\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_729\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_730\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_223\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4asbjgpobnapsag\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_731\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_732\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4asbjgpobnapsag","anchor":"9327957"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ub7ro3bd77y6ze5\/","ref":"T345811#9327983"},[],{"anchor":"9327983"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_225\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_738\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_739\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_227\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ub7ro3bd77y6ze5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_740\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_741\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ub7ro3bd77y6ze5","anchor":"9327983"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-etqld2dk3phqvxl\/","ref":"T345811#9327984"},[],{"anchor":"9327984"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_229\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_747\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_748\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_231\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-etqld2dk3phqvxl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_749\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_750\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-etqld2dk3phqvxl","anchor":"9327984"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-e3v4xlefzopc4oy\/","ref":"T345811#9327990"},[],{"anchor":"9327990"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_233\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_756\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_757\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_235\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-e3v4xlefzopc4oy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_758\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_759\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-e3v4xlefzopc4oy","anchor":"9327990"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ojnk3b5comhumj4\/","ref":"T345811#9328047"},[],{"anchor":"9328047"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_237\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_765\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_766\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_239\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ojnk3b5comhumj4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_767\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_768\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ojnk3b5comhumj4","anchor":"9328047"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fsxyivz7lwuexa6\/","ref":"T345811#9328076"},[],{"anchor":"9328076"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_241\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_774\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_775\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_243\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fsxyivz7lwuexa6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_776\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_777\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fsxyivz7lwuexa6","anchor":"9328076"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fhj5tlgxf2snszd\/","ref":"T345811#9328077"},[],{"anchor":"9328077"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_245\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_783\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_784\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_247\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fhj5tlgxf2snszd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_785\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_786\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fhj5tlgxf2snszd","anchor":"9328077"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fhsg4c2w4r3l2ib\/","ref":"T345811#9328084"},[],{"anchor":"9328084"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_249\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_792\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_793\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_251\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fhsg4c2w4r3l2ib\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_794\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_795\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fhsg4c2w4r3l2ib","anchor":"9328084"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3o2vlihb4bgbjdg\/","ref":"T345811#9328130"},[],{"anchor":"9328130"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_253\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_801\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_802\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_255\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3o2vlihb4bgbjdg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_803\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_804\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3o2vlihb4bgbjdg","anchor":"9328130"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7pv6euzdn42y7tk\/","ref":"T345811#9328175"},[],{"anchor":"9328175"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_257\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_810\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_811\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_259\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7pv6euzdn42y7tk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_812\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_813\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7pv6euzdn42y7tk","anchor":"9328175"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3l72z2iuln3z7g7\/","ref":"T345811#9328179"},[],{"anchor":"9328179"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_261\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_819\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_820\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_263\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3l72z2iuln3z7g7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_821\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_822\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3l72z2iuln3z7g7","anchor":"9328179"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-eu6fpxxdxr7mcqa\/","ref":"T345811#9328246"},[],{"anchor":"9328246"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_265\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_828\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_829\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_267\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eu6fpxxdxr7mcqa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_830\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_831\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-eu6fpxxdxr7mcqa","anchor":"9328246"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-as2vjm2udhji6cx\/","ref":"T345811#9328293"},[],{"anchor":"9328293"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_269\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_837\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_838\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_271\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-as2vjm2udhji6cx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_839\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_840\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-as2vjm2udhji6cx","anchor":"9328293"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5jh4kwvc4si7v7c\/","ref":"T345811#9328296"},[],{"anchor":"9328296"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_273\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_846\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_847\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_275\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5jh4kwvc4si7v7c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_848\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_849\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5jh4kwvc4si7v7c","anchor":"9328296"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wscgiq6cooi5lfa\/","ref":"T345811#9328297"},[],{"anchor":"9328297"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_277\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_855\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_856\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_279\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wscgiq6cooi5lfa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_857\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_858\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wscgiq6cooi5lfa","anchor":"9328297"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ofwka6eibdp7c74\/","ref":"T345811#9328301"},[],{"anchor":"9328301"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_281\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_864\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_865\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_283\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ofwka6eibdp7c74\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_866\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_867\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ofwka6eibdp7c74","anchor":"9328301"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7ku44onlnsp453u\/","ref":"T345811#9328307"},[],{"anchor":"9328307"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_285\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_873\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_874\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_287\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7ku44onlnsp453u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_875\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_876\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7ku44onlnsp453u","anchor":"9328307"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6m336cwhhi5qkv5\/","ref":"T345811#9328335"},[],{"anchor":"9328335"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_289\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_882\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_883\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_291\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6m336cwhhi5qkv5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_884\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_885\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6m336cwhhi5qkv5","anchor":"9328335"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-orqpewtplpvzdvs\/","ref":"T345811#9328351"},[],{"anchor":"9328351"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_293\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_891\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_892\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_295\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-orqpewtplpvzdvs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_893\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_894\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-orqpewtplpvzdvs","anchor":"9328351"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-34pjmpaz242vgey\/","ref":"T345811#9328354"},[],{"anchor":"9328354"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_297\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_900\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_901\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_299\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-34pjmpaz242vgey\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_902\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_903\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-34pjmpaz242vgey","anchor":"9328354"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uyqezligowagvft\/","ref":"T345811#9328452"},[],{"anchor":"9328452"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_301\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_909\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_910\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_303\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uyqezligowagvft\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_911\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_912\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uyqezligowagvft","anchor":"9328452"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mnfo5k7unmkazbh\/","ref":"T345811#9328476"},[],{"anchor":"9328476"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_305\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_918\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_919\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_307\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mnfo5k7unmkazbh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_920\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_921\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mnfo5k7unmkazbh","anchor":"9328476"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dbybxohqf7skket\/","ref":"T345811#9328946"},[],{"anchor":"9328946"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_309\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_927\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_928\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_311\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dbybxohqf7skket\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_929\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_930\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dbybxohqf7skket","anchor":"9328946"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3he2lj6pprcrpwm\/","ref":"T345811#9329027"},[],{"anchor":"9329027"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_313\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_936\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_937\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_315\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3he2lj6pprcrpwm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_938\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_939\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3he2lj6pprcrpwm","anchor":"9329027"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bpejlmeookin4xl\/","ref":"T345811#9329045"},[],{"anchor":"9329045"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_317\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_945\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_946\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_319\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bpejlmeookin4xl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_947\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_948\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bpejlmeookin4xl","anchor":"9329045"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-i7foqmt7fj32rmp\/","ref":"T345811#9329077"},[],{"anchor":"9329077"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_321\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_954\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_955\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_323\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-i7foqmt7fj32rmp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_956\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_957\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-i7foqmt7fj32rmp","anchor":"9329077"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-znawqneokj2mrjx\/","ref":"T345811#9329104"},[],{"anchor":"9329104"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_325\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_963\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_964\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_327\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-znawqneokj2mrjx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_965\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_966\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-znawqneokj2mrjx","anchor":"9329104"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ak2zyn7u6ez7btk\/","ref":"T345811#9329143"},[],{"anchor":"9329143"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_329\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_972\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_973\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_331\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ak2zyn7u6ez7btk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_974\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_975\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ak2zyn7u6ez7btk","anchor":"9329143"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-z6o44xdgmwayqxr\/","ref":"T345811#9329144"},[],{"anchor":"9329144"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_333\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_981\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_982\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_335\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-z6o44xdgmwayqxr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_983\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_984\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-z6o44xdgmwayqxr","anchor":"9329144"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-egegman4ozic2iu\/","ref":"T345811#9329145"},[],{"anchor":"9329145"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_337\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_990\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_991\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_339\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-egegman4ozic2iu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_992\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_993\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-egegman4ozic2iu","anchor":"9329145"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-x4ktdmn7abmcbrw\/","ref":"T345811#9329157"},[],{"anchor":"9329157"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_341\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_999\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1000\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_343\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-x4ktdmn7abmcbrw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1001\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1002\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-x4ktdmn7abmcbrw","anchor":"9329157"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zuqxnsgnzpqfla2\/","ref":"T345811#9329158"},[],{"anchor":"9329158"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_345\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1008\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1009\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_347\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zuqxnsgnzpqfla2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1010\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1011\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zuqxnsgnzpqfla2","anchor":"9329158"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2tfjtchji6x2gvy\/","ref":"T345811#9329160"},[],{"anchor":"9329160"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_349\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1017\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1018\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_351\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2tfjtchji6x2gvy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1019\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1020\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2tfjtchji6x2gvy","anchor":"9329160"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kbcvl5radm4hwsd\/","ref":"T345811#9329177"},[],{"anchor":"9329177"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_353\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1026\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1027\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_355\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kbcvl5radm4hwsd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1028\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1029\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kbcvl5radm4hwsd","anchor":"9329177"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-aayl2arp6qhs2ol\/","ref":"T345811#9329186"},[],{"anchor":"9329186"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_357\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1035\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1036\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_359\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aayl2arp6qhs2ol\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1037\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1038\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-aayl2arp6qhs2ol","anchor":"9329186"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mjshsmks6ws4aud\/","ref":"T345811#9329192"},[],{"anchor":"9329192"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_361\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1044\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1045\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_363\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mjshsmks6ws4aud\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1046\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1047\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mjshsmks6ws4aud","anchor":"9329192"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wph2wulbtwesuqo\/","ref":"T345811#9329194"},[],{"anchor":"9329194"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_365\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1053\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1054\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_367\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wph2wulbtwesuqo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1055\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1056\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wph2wulbtwesuqo","anchor":"9329194"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7lgbfh3aven2ntq\/","ref":"T345811#9329197"},[],{"anchor":"9329197"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_369\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1062\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1063\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_371\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7lgbfh3aven2ntq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1064\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1065\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7lgbfh3aven2ntq","anchor":"9329197"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xo2qldz4vn354y4\/","ref":"T345811#9329198"},[],{"anchor":"9329198"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_373\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1071\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1072\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_375\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xo2qldz4vn354y4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1073\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1074\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xo2qldz4vn354y4","anchor":"9329198"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kgrtnrzyqpedk7p\/","ref":"T345811#9329200"},[],{"anchor":"9329200"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_377\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1080\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1081\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_379\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kgrtnrzyqpedk7p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1082\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1083\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kgrtnrzyqpedk7p","anchor":"9329200"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zejup2ht3i76xij\/","ref":"T345811#9329201"},[],{"anchor":"9329201"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_381\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1089\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1090\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_383\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zejup2ht3i76xij\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1091\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1092\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zejup2ht3i76xij","anchor":"9329201"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-enjs32bm646briv\/","ref":"T345811#9329203"},[],{"anchor":"9329203"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_385\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1098\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1099\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_387\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-enjs32bm646briv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1100\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1101\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-enjs32bm646briv","anchor":"9329203"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rnggxtmym65jntx\/","ref":"T345811#9329204"},[],{"anchor":"9329204"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_389\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1107\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1108\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_391\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rnggxtmym65jntx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1109\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1110\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rnggxtmym65jntx","anchor":"9329204"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rq5usdeo6x4keqc\/","ref":"T345811#9329228"},[],{"anchor":"9329228"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_393\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1116\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1117\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_395\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rq5usdeo6x4keqc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1118\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1119\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rq5usdeo6x4keqc","anchor":"9329228"}],"javelin_behaviors":{"phui-hovercards":[],"repository-crossreference":[],"phui-dropdown-menu":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cc8a2c6b\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/b37bcd38\/rsrc\/css\/application\/paste\/paste.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/e382316a\/rsrc\/css\/layout\/phabricator-source-code-view.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/46fcb3af\/differential.pkg.js"]}