for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_199\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/263tz2pyjwqwewbq4ct7\/PHID-FILE-oktdhozw4xeg4efc4zmf\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bking\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9479197\" id=\"9479197\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003ebking\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9479197\" data-sigil=\"has-tooltip\" data-meta=\"0_196\"\u003e\u003cspan class=\"screen-only\"\u003eJan 22 2024, 11:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-22 23:08:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_198\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003ebking\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-n5ll5jsqwlfm7mk\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/263tz2pyjwqwewbq4ct7\/PHID-FILE-oktdhozw4xeg4efc4zmf\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bking\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9481098\" id=\"9481098\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003ebking\u003c\/a\u003e mentioned this in \u003ca href=\"\/T346946\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003eT346946: move cloudelastic behind cloudlb\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9481098\" data-sigil=\"has-tooltip\" data-meta=\"0_200\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 2:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 14:07:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_205\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wxni4dd2w73zpqvtmjdq\/PHID-FILE-ywrfhopc4oo74akgzhxh\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gehel\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9481322\" id=\"9481322\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_204\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gehel\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eGehel\u003c\/a\u003e edited projects, added \u003ca href=\"\/project\/view\/6928\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eData-Platform-SRE (2024.01.22 - 2024.02.11)\u003c\/a\u003e; removed \u003ca href=\"\/tag\/data-platform-sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003eData-Platform-SRE\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9481322\" data-sigil=\"has-tooltip\" data-meta=\"0_203\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 2:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 14:44:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_214\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xh2ph2yp26zepxxurglq\/PHID-FILE-w7ofzuyouv2jmz2zuzuo\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cmooney\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9481488\" id=\"9481488\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_213\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003ecmooney\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9481488\" data-sigil=\"has-tooltip\" data-meta=\"0_212\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 3:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 15:11:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_210\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_211\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bking\u003c\/span\u003e\u003c\/a\u003e what is the current status of \u003ca href=\"\/T351354\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT351354: Service implementation for cloudelastic1007-1010\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e ?\u003c\/p\u003e\n\n\u003cp\u003eIf those new nodes are not live yet we can hopefully move them all to private IPs before they are serving traffic, which might make the move easier?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_217\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/263tz2pyjwqwewbq4ct7\/PHID-FILE-oktdhozw4xeg4efc4zmf\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bking\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9482519\" id=\"9482519\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003ebking\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/6928\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/6928\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003eIn Progress\u003c\/a\u003e on the \u003ca href=\"\/project\/view\/6928\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eData-Platform-SRE (2024.01.22 - 2024.02.11)\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9482519\" data-sigil=\"has-tooltip\" data-meta=\"0_215\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 7:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 19:26:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_228\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/263tz2pyjwqwewbq4ct7\/PHID-FILE-oktdhozw4xeg4efc4zmf\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bking\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/22\/\" data-sigil=\"has-tooltip\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-graduation-cap\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9482761\" id=\"9482761\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003ebking\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9482761\" data-sigil=\"has-tooltip\" data-meta=\"0_224\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 9:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 21:06:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_222\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_78\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cmooney\u003c\/span\u003e\u003c\/a\u003e Unfortunately, cloudelastic is a stateful, clustered service so we'll have to worry about quorum and split-brain. I'm \u003ca href=\"https:\/\/etherpad.wikimedia.org\/p\/cloudelastic-T355617\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eputting together a maintenance plan\u003c\/a\u003e but it's not quite ready yet.\u003c\/p\u003e\n\n\u003cp\u003eI did have a question about \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Server_Lifecycle#Move_existing_server_between_rows\/racks,_changing_IPs\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Server_Lifecycle#Move_existing_server_between_rows\/racks,_changing_IPs\u003c\/a\u003e (\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e 's recommended procedure). Is there a way to do this without data loss? The cloudelastic hosts have ~1.5 TB of data; backing up\/restoring that data would add days to the downtime. Maybe something closer to the maintenance plan y'all helped with a while back would work? (ref \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Server_Lifecycle#Change_server_NIC_and_switch_connection,_keeping_IPs\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Server_Lifecycle#Change_server_NIC_and_switch_connection,_keeping_IPs\u003c\/a\u003e )\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_237\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xh2ph2yp26zepxxurglq\/PHID-FILE-w7ofzuyouv2jmz2zuzuo\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cmooney\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9482787\" id=\"9482787\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003ecmooney\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9482787\" data-sigil=\"has-tooltip\" data-meta=\"0_235\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 9:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 21:22:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_233\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_80\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T355617#9482761\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617#9482761\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/bking\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bking\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cmooney\u003c\/span\u003e\u003c\/a\u003e Unfortunately, cloudelastic is a stateful, clustered service so we'll have to worry about quorum and split-brain\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIs there a reason the nodes can't form a quorum with some of them on a public IP and some of them on a private IP?  I can definitely see how that would complicate matters a lot, but I'm not sure that's the case?\u003c\/p\u003e\n\n\u003cp\u003eIn my naivety I figured we could bring 1007-1010 into the cluster (on private IPs from the start), and then once they have synced sufficient data "depool" the others 1 at a time, moving IP address and then re-adding.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eMaybe something closer to the maintenance plan y'all helped with a while back would work?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAbsolutely, we pushed you to the reimage link with the new servers in mind.  But for the existing ones if they have lots of data it does sound like we should change the IP rather than reimage.  We can follow the process you list, but you can leave the Netbox\/DNS\/switch config side of it to us.  Once it's safe to do so we then just change the \/etc\/network\/interfaces config file manually on the host and reboot.\u003c\/p\u003e\n\n\u003cp\u003eThe other obvious question we need to ask about the IP change is how the nodes identify each other?\u003c\/p\u003e\n\n\u003cp\u003eIf the config has hard-coded IPs we'll obviously need to change those as they move.  Often DNS saves us if that's what's hard-coded in some config, but problem here is the fqdn for hosts will change from (for example), \u003cem\u003ecloudelastic1006.wikimedia.org\u003c\/em\u003e to \u003cem\u003ecloudelastic1006.eqiad.wmnet\u003c\/em\u003e.  Basically we need to change any mention of the old IP\/hostname on any device for the new one.  Usually not an issue but some applications are awkward.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_246\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9483224\" id=\"9483224\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9483224\" data-sigil=\"has-tooltip\" data-meta=\"0_244\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 11:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 23:47:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_242\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 992547 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: Configure post-migration TLS\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/992547\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/992547\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_249\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9483225\" id=\"9483225\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9483225\" data-sigil=\"has-tooltip\" data-meta=\"0_247\"\u003e\u003cspan class=\"screen-only\"\u003eJan 23 2024, 11:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-23 23:47:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_260\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/263tz2pyjwqwewbq4ct7\/PHID-FILE-oktdhozw4xeg4efc4zmf\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bking\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/22\/\" data-sigil=\"has-tooltip\" data-meta=\"0_258\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-graduation-cap\" data-meta=\"0_259\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9483282\" id=\"9483282\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003ebking\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9483282\" data-sigil=\"has-tooltip\" data-meta=\"0_256\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2024, 12:10 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-24 00:10:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_254\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T355617#9482787\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617#9482787\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/cmooney\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cmooney\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T355617#9482761\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617#9482761\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/bking\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bking\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cmooney\u003c\/span\u003e\u003c\/a\u003e Unfortunately, cloudelastic is a stateful, clustered service so we'll have to worry about quorum and split-brain\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cmooney\u003c\/span\u003e\u003c\/a\u003e and I discussed this off-task, updating to reflect our conversation.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIs there a reason the nodes can't form a quorum with some of them on a public IP and some of them on a private IP?  I can definitely see how that would complicate matters a lot, but I'm not sure that's the case?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSorry, you are correct: there's nothing intrinsic to Elastic that would prevent public and private IPs from joining a quorum. Which makes the maintenance plan much simpler. Thanks for asking that, I wouldn't have snapped to it if not ;)\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eMaybe something closer to the maintenance plan y'all helped with a while back would work?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAbsolutely, we pushed you to the reimage link with the new servers in mind.  But for the existing ones if they have lots of data it does sound like we should change the IP rather than reimage.  We can follow the process you list, but you can leave the Netbox\/DNS\/switch config side of it to us.  Once it's safe to do so we then just change the \/etc\/network\/interfaces config file manually on the host and reboot.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI'd prefer to do it that way, but since I now understand we can cluster together public and private IPs, reimaging is perfectly acceptable. Although if we are going to work together synchronously, the IP changes would be faster as we wouldn't have to wait for the new hosts to pull down their full 1.5TB data set.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eproblem here is the fqdn for hosts will change from (for example), \u003cem\u003ecloudelastic1006.wikimedia.org\u003c\/em\u003e to \u003cem\u003ecloudelastic1006.eqiad.wmnet\u003c\/em\u003e.  Basically we need to change any mention of the old IP\/hostname on any device for the new one.  Usually not an issue but some applications are awkward.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThanks, this has more implications for TLS than for cluster communication. I've spun out \u003ca href=\"\/T355720\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355720\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e for the TLS needs. Will ping you back when we're ready to test our canary.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_271\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/32ma7fpuzxevmzjow4hj\/PHID-FILE-ejkcs7pcsa3shp324bda\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_269\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_270\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9484282\" id=\"9484282\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_268\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_87\"\u003etaavi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9484282\" data-sigil=\"has-tooltip\" data-meta=\"0_267\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2024, 10:53 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-24 10:53:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_265\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_266\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_88\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eTrying to change the node FQDN without a re-image seems more trouble than it's worth if that can be avoided relatively easily (which it seems like we can?).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_280\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9485627\" id=\"9485627\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_279\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_89\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9485627\" data-sigil=\"has-tooltip\" data-meta=\"0_278\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2024, 4:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-24 16:02:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_276\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_277\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_90\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 992748 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: lay the groundwork for private IP migration\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/992748\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/992748\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_283\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2lgx6x2eyk2tyor5xik2\/PHID-FILE-pqoa3pykad5w4ljqx466\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EBernhardson\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9486083\" id=\"9486083\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EBernhardson\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003eEBernhardson\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9486083\" data-sigil=\"has-tooltip\" data-meta=\"0_281\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2024, 5:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-24 17:51:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_287\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xh2ph2yp26zepxxurglq\/PHID-FILE-w7ofzuyouv2jmz2zuzuo\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cmooney\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9487945\" id=\"9487945\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003ecmooney\u003c\/a\u003e added a subtask: \u003ca href=\"\/T355862\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003eT355862: Migrate servers in codfw rack A3 from asw-a3-codfw to lsw1-a3-codfw\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9487945\" data-sigil=\"has-tooltip\" data-meta=\"0_284\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 11:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 11:37:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_286\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003ecmooney\u003c\/a\u003e added a subtask: \u003ca href=\"\/T355861\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_95\"\u003eT355861: Migrate servers in codfw rack A2 from asw-a2-codfw to lsw1-a2-codfw\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_291\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xh2ph2yp26zepxxurglq\/PHID-FILE-w7ofzuyouv2jmz2zuzuo\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cmooney\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9487964\" id=\"9487964\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003ecmooney\u003c\/a\u003e removed a subtask: \u003ca href=\"\/T355861\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003eT355861: Migrate servers in codfw rack A2 from asw-a2-codfw to lsw1-a2-codfw\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9487964\" data-sigil=\"has-tooltip\" data-meta=\"0_288\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 11:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 11:40:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_290\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003ecmooney\u003c\/a\u003e removed a subtask: \u003ca href=\"\/T355862\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_99\"\u003eT355862: Migrate servers in codfw rack A3 from asw-a3-codfw to lsw1-a3-codfw\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_300\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489194\" id=\"9489194\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_299\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489194\" data-sigil=\"has-tooltip\" data-meta=\"0_298\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 6:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 18:18:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_296\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_297\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_101\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993014 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/dns@master] cloudelastic: add CNAME for migration canary\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993014\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993014\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_303\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/e7ctcsffgw7u5swqxel2\/PHID-FILE-etgj5apgjf7g47rzgoes\/1db5c7-alphanumeric_lato-white_R.png-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/RKemper\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"9489591\" id=\"9489591\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_302\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RKemper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003eRKemper\u003c\/a\u003e mentioned this in \u003ca href=\"\/T351354\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_103\"\u003eT351354: Service implementation for cloudelastic1007-1010\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489591\" data-sigil=\"has-tooltip\" data-meta=\"0_301\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:48:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_312\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489596\" id=\"9489596\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489596\" data-sigil=\"has-tooltip\" data-meta=\"0_310\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:52:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_308\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_309\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/3dzTQo0BGiVuUzOdT6NL\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T22:52:41Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.ban Banning hosts: cloudelastic1010 for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_321\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489597\" id=\"9489597\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489597\" data-sigil=\"has-tooltip\" data-meta=\"0_319\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:52:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_317\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_318\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/4NzTQo0BGiVuUzOdYqOK\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T22:52:46Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.ban (exit_code=99) Banning hosts: cloudelastic1010 for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_330\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489598\" id=\"9489598\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_329\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489598\" data-sigil=\"has-tooltip\" data-meta=\"0_328\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:53:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_326\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_327\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_109\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/YbXUQo0BhuQtenzvU2_z\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_4\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T22:53:47Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.ban Banning hosts: cloudelastic1010 for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_339\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489599\" id=\"9489599\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_338\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489599\" data-sigil=\"has-tooltip\" data-meta=\"0_337\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:53:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_335\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_111\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/yg7UQo0BxE1_1c7sZL3a\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_5\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T22:53:52Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.ban (exit_code=99) Banning hosts: cloudelastic1010 for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_348\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489600\" id=\"9489600\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_347\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489600\" data-sigil=\"has-tooltip\" data-meta=\"0_346\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:53:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_344\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_345\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_113\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/bLXUQo0BhuQtenzvdW8z\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_6\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T22:53:56Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.ban Banning hosts: cloudelastic1010.wikimedia.org for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_357\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489601\" id=\"9489601\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489601\" data-sigil=\"has-tooltip\" data-meta=\"0_355\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 10:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 22:54:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_353\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_354\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/2g7UQo0BxE1_1c7shb2W\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_7\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T22:54:00Z] <bking@cumin2002> END (PASS) - Cookbook sre.elasticsearch.ban (exit_code=0) Banning hosts: cloudelastic1010.wikimedia.org for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_366\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489634\" id=\"9489634\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489634\" data-sigil=\"has-tooltip\" data-meta=\"0_364\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 11:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 23:17:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_362\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_363\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/Hg7qQo0BxE1_1c7sMs1p\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_8\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T23:17:41Z] <bking@cumin2002> START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on cloudelastic1010.wikimedia.org with reason: migration canary \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_375\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9489635\" id=\"9489635\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9489635\" data-sigil=\"has-tooltip\" data-meta=\"0_373\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2024, 11:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-25 23:17:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_371\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_372\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_119\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/IQ7qQo0BxE1_1c7scM0M\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_9\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-25T23:17:56Z] <bking@cumin2002> END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on cloudelastic1010.wikimedia.org with reason: migration canary \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_384\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9490902\" id=\"9490902\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_120\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9490902\" data-sigil=\"has-tooltip\" data-meta=\"0_382\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 3:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 15:06:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_380\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993103 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: use CFSSL for TLS on canary\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993103\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993103\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_393\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9490988\" id=\"9490988\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9490988\" data-sigil=\"has-tooltip\" data-meta=\"0_391\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 3:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 15:27:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_389\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993148 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: apply cloudelastic role to canary\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993148\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993148\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_402\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491010\" id=\"9491010\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491010\" data-sigil=\"has-tooltip\" data-meta=\"0_400\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 3:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 15:33:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_398\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 992547 \u003cstrong\u003emerged\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: config changes for migration canary\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/992547\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/992547\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_411\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491031\" id=\"9491031\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491031\" data-sigil=\"has-tooltip\" data-meta=\"0_409\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 3:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 15:42:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_407\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_408\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_127\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993150 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: remove references to cloudelastic1010\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993150\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993150\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_420\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491088\" id=\"9491088\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_128\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491088\" data-sigil=\"has-tooltip\" data-meta=\"0_418\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 4:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 16:03:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_416\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_417\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_129\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993150 \u003cstrong\u003emerged\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: remove references to cloudelastic1010\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993150\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993150\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_429\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491108\" id=\"9491108\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491108\" data-sigil=\"has-tooltip\" data-meta=\"0_427\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 4:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 16:15:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_425\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/wBGORo0BxE1_1c7sWu5H\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_10\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-26T16:15:50Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_438\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491149\" id=\"9491149\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491149\" data-sigil=\"has-tooltip\" data-meta=\"0_436\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 4:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 16:33:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_434\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/LhGeRo0BxE1_1c7sNPaF\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_11\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-26T16:33:09Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.rolling-operation (exit_code=99) Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_447\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491150\" id=\"9491150\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_446\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491150\" data-sigil=\"has-tooltip\" data-meta=\"0_445\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 4:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 16:33:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_443\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.hosts.decommission executed by bking@cumin2002 for hosts: \u003ctt class=\"remarkup-monospaced\"\u003ecloudelastic1010.wikimedia.org\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudelastic1010.wikimedia.org (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound physical host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed management interface on Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWiped all swraid, partition-table and filesystem signatures\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePowered off\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e[Netbox] Set status to Decommissioning, deleted all non-mgmt IPs,  updated switch interfaces (disabled, removed vlans, etc)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConfigured the linked switch interface(s)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from DebMonitor\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet master and PuppetDB\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_456\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491260\" id=\"9491260\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491260\" data-sigil=\"has-tooltip\" data-meta=\"0_454\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 5:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 17:17:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_452\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage was started by bking@cumin2002 for host cloudelastic1010.eqiad.wmnet with OS bullseye\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_465\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9491332\" id=\"9491332\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_464\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9491332\" data-sigil=\"has-tooltip\" data-meta=\"0_463\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2024, 5:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-26 17:57:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_461\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_462\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCookbook cookbooks.sre.hosts.reimage started by bking@cumin2002 for host cloudelastic1010.eqiad.wmnet with OS bullseye completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecloudelastic1010 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Puppet and PuppetDB if present and deleted any certificates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoved from Debmonitor if present\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced PXE for next reboot\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost rebooted via IPMI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (Debian installer)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdd puppet_version metadata to Debian installer\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChecked BIOS boot parameters are back to normal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHost up (new fresh bullseye OS)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerated Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSigned new Puppet certificate\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRun Puppet in NOOP mode to populate exported resources in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFound Nagios_host resource for this host in PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDowntimed the new host on Icinga\/Alertmanager\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFirst Puppet run completed and logged in \/var\/log\/spicerack\/sre\/hosts\/reimage\/202401261737_bking_2287993_cloudelastic1010.out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigmaster.wikimedia.org updated with the host new SSH public key for wmf-update-known-hosts-production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRebooted\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatic Puppet run was successful\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eForced a re-check of all Icinga services for the host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga status is optimal\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIcinga downtime removed\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox data from PuppetDB\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUpdated Netbox status planned -> active\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe sre.puppet.sync-netbox-hiera cookbook was run successfully\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_474\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xh2ph2yp26zepxxurglq\/PHID-FILE-w7ofzuyouv2jmz2zuzuo\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cmooney\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9493745\" id=\"9493745\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_473\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003ecmooney\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9493745\" data-sigil=\"has-tooltip\" data-meta=\"0_472\"\u003e\u003cspan class=\"screen-only\"\u003eJan 29 2024, 12:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-29 12:50:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_470\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_471\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/bking\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bking\u003c\/span\u003e\u003c\/a\u003e I see that cloudelastic1010 seems to be happy on it's new IP\/hostname?  Glad that it seems to have gone well.  I am mostly on leave this week so may not be on irc, but I'll be checking in here and there so let me know on the task if there are any glitches.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_483\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9495837\" id=\"9495837\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_482\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_142\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9495837\" data-sigil=\"has-tooltip\" data-meta=\"0_481\"\u003e\u003cspan class=\"screen-only\"\u003eJan 29 2024, 7:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-29 19:18:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_479\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_480\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993764 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: Add migration canary to cloudelastic cluster\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993764\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993764\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_492\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9495846\" id=\"9495846\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_491\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_144\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9495846\" data-sigil=\"has-tooltip\" data-meta=\"0_490\"\u003e\u003cspan class=\"screen-only\"\u003eJan 29 2024, 7:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-29 19:21:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_488\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_489\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_145\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993148 \u003cstrong\u003eabandoned\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: apply cloudelastic role to canary\u003c\/p\u003e\n\n\u003cp\u003eReason:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003esuperseded by I0627fb939764e5bd12156d1bffb410e3732e36ba\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993148\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993148\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_501\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9495849\" id=\"9495849\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_500\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_146\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9495849\" data-sigil=\"has-tooltip\" data-meta=\"0_499\"\u003e\u003cspan class=\"screen-only\"\u003eJan 29 2024, 7:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-29 19:21:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_497\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_498\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_147\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993103 \u003cstrong\u003eabandoned\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: use CFSSL for TLS on canary\u003c\/p\u003e\n\n\u003cp\u003eReason:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003esuperseded by I0627fb939764e5bd12156d1bffb410e3732e36ba\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993103\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993103\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_510\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9498724\" id=\"9498724\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_509\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_148\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9498724\" data-sigil=\"has-tooltip\" data-meta=\"0_508\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 3:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 15:22:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_506\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_507\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_149\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993764 \u003cstrong\u003emerged\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: Add migration canary to cloudelastic cluster\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993764\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993764\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_519\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9498806\" id=\"9498806\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_518\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_150\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9498806\" data-sigil=\"has-tooltip\" data-meta=\"0_517\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 3:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 15:40:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_515\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_516\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_151\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/myUHW40BxE1_1c7s0taL\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_12\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T15:40:55Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_528\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9498868\" id=\"9498868\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_527\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_152\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9498868\" data-sigil=\"has-tooltip\" data-meta=\"0_526\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 3:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 15:54:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_524\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_525\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/RyUUW40BxE1_1c7saOB6\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_13\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T15:54:40Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.rolling-operation (exit_code=99) Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_537\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9498877\" id=\"9498877\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9498877\" data-sigil=\"has-tooltip\" data-meta=\"0_535\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 15:58:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_533\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_534\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_155\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/3iUXW40BxE1_1c7slOVi\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_14\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T15:58:08Z] <bking@cumin2002> START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on cloudelastic1010.eqiad.wmnet with reason: \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_546\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9498879\" id=\"9498879\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_545\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_156\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9498879\" data-sigil=\"has-tooltip\" data-meta=\"0_544\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 15:58:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_542\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_543\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_157\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/o84XW40BhuQtenzv0yAC\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_15\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T15:58:24Z] <bking@cumin2002> END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on cloudelastic1010.eqiad.wmnet with reason: \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_555\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9498940\" id=\"9498940\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_554\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_158\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9498940\" data-sigil=\"has-tooltip\" data-meta=\"0_553\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 4:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 16:14:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_551\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_552\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_159\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/UiUmW40BxE1_1c7suPUX\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_16\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T16:14:40Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_564\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499044\" id=\"9499044\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_563\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_160\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499044\" data-sigil=\"has-tooltip\" data-meta=\"0_562\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 4:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 16:34:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_560\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_561\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/o_U4W40BGiVuUzOdalmq\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_17\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T16:34:00Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.rolling-operation (exit_code=99) Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_573\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499184\" id=\"9499184\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_572\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_162\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499184\" data-sigil=\"has-tooltip\" data-meta=\"0_571\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 4:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 16:57:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_569\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_570\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_163\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/N85NW40BhuQtenzvhV2K\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_18\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T16:57:03Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_582\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499340\" id=\"9499340\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_581\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_164\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499340\" data-sigil=\"has-tooltip\" data-meta=\"0_580\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 5:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 17:37:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_578\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_579\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_165\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/zPVyW40BGiVuUzOd2J4P\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_19\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T17:37:49Z] <bking@cumin2002> END (PASS) - Cookbook sre.elasticsearch.rolling-operation (exit_code=0) Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate new elastic config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_591\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499827\" id=\"9499827\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_590\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_166\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499827\" data-sigil=\"has-tooltip\" data-meta=\"0_589\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 7:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 19:36:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_587\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_588\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_167\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/KibfW40BxE1_1c7scrfr\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_20\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T19:36:27Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.ban Banning hosts: cloudelastic1010.eqiad.wmnet for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_600\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499828\" id=\"9499828\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_599\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_168\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499828\" data-sigil=\"has-tooltip\" data-meta=\"0_598\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 7:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 19:36:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_596\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_597\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/1_bfW40BGiVuUzOdhhjH\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_21\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T19:36:32Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.ban (exit_code=99) Banning hosts: cloudelastic1010.eqiad.wmnet for use cloudelastic1010 as migration canary - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_609\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499890\" id=\"9499890\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_608\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_170\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499890\" data-sigil=\"has-tooltip\" data-meta=\"0_607\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 8:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 20:01:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_605\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_606\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994251 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: listen on "site" scoped IPs as well\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994251\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994251\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_618\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9499897\" id=\"9499897\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_617\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_172\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9499897\" data-sigil=\"has-tooltip\" data-meta=\"0_616\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 8:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 20:07:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_614\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_615\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_173\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994251 \u003cstrong\u003emerged\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: listen on "site" scoped IPs as well\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994251\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994251\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_627\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500156\" id=\"9500156\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_626\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_174\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500156\" data-sigil=\"has-tooltip\" data-meta=\"0_625\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 8:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 20:51:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_623\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_624\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_175\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/OvYkXI0BGiVuUzOdDV89\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_22\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T20:51:22Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate first private IP host config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_636\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500164\" id=\"9500164\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_635\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_176\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500164\" data-sigil=\"has-tooltip\" data-meta=\"0_634\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 8:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 20:52:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_632\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_633\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/jyYlXI0BxE1_1c7sa_om\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_23\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T20:52:52Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.rolling-operation (exit_code=99) Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate first private IP host config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_645\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500192\" id=\"9500192\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_644\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_178\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500192\" data-sigil=\"has-tooltip\" data-meta=\"0_643\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 8:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 20:59:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_641\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_642\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994321 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: bind to all interfaces\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994321\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994321\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_654\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500274\" id=\"9500274\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_653\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500274\" data-sigil=\"has-tooltip\" data-meta=\"0_652\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 9:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 21:14:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_650\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_651\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994321 \u003cstrong\u003emerged\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: bind to all interfaces\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994321\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994321\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_663\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500310\" id=\"9500310\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_662\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500310\" data-sigil=\"has-tooltip\" data-meta=\"0_661\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 9:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 21:34:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_659\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_660\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_183\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/hM9LXI0BhuQtenzvkmQ-\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_24\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T21:34:32Z] <bking@cumin2002> START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate first private IP host config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_672\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500403\" id=\"9500403\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_671\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_184\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500403\" data-sigil=\"has-tooltip\" data-meta=\"0_670\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 9:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 21:54:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_668\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_669\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_185\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 992748 \u003cstrong\u003eabandoned\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: enable DNS discovery\/VIP for test service\u003c\/p\u003e\n\n\u003cp\u003eReason:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003eWe are not going to use discovery service after all\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/992748\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/992748\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_681\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500404\" id=\"9500404\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_680\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_186\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500404\" data-sigil=\"has-tooltip\" data-meta=\"0_679\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 9:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 21:54:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_677\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_187\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 993014 \u003cstrong\u003eabandoned\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/dns@master] cloudelastic: add CNAME for migration canary\u003c\/p\u003e\n\n\u003cp\u003eReason:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003ewe are not going to use service discovery after all\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/993014\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/993014\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_690\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500535\" id=\"9500535\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_689\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_188\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500535\" data-sigil=\"has-tooltip\" data-meta=\"0_688\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 10:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 22:39:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_686\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_687\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994338 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: use acme-chief\/letsencrypt with canary\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994338\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994338\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_699\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9500537\" id=\"9500537\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_698\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_190\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9500537\" data-sigil=\"has-tooltip\" data-meta=\"0_697\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2024, 10:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-30 22:40:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_695\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_696\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_191\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/X8-IXI0BhuQtenzvYp82\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_25\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2024-01-30T22:40:58Z] <bking@cumin2002> END (FAIL) - Cookbook sre.elasticsearch.rolling-operation (exit_code=99) Operation.RESTART (1 nodes at a time) for ElasticSearch cluster cloudelastic: activate first private IP host config - bking@cumin2002 - \u003ca href=\"\/T355617\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT355617\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_708\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9502766\" id=\"9502766\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_707\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_192\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9502766\" data-sigil=\"has-tooltip\" data-meta=\"0_706\"\u003e\u003cspan class=\"screen-only\"\u003eJan 31 2024, 4:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-31 16:31:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_704\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_705\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_193\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994338 \u003cstrong\u003emerged\u003c\/strong\u003e by Bking:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: use acme-chief\/letsencrypt with canary\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994338\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994338\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_717\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"9502848\" id=\"9502848\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_716\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#9502848\" data-sigil=\"has-tooltip\" data-meta=\"0_715\"\u003e\u003cspan class=\"screen-only\"\u003eJan 31 2024, 4:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-01-31 16:40:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_713\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_195\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 994763 had a related patch set uploaded (by Bking; author: Bking):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cloudelastic: allow wmnet hosts to request certs from acme-chief\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/994763\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/994763\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],{"hovercardSpec":{"objectPHID":"PHID-TASK-ybearlyqae26w7cbuela"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-muqtoci52ojpxbj5yewp"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r","contextPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ehyjtijp5zgb6xf4drz4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-wirykp3hlzatuqiofcgi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-oknilgmxl4rosyulvny7"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-qvokajx3knjtqh7nnieh"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-r456pnp5exj6uphuhwy6"}},{"phid":"PHID-XACT-TASK-wfg7etakjundamm"},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-my7pddhllyrh3wjng52j"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-z2rxei3p7asjbotkzpvf"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-qvokajx3knjtqh7nnieh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r"}},{"phid":"PHID-XACT-TASK-hn5kcx4vjaniu3d"},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"phid":"PHID-XACT-TASK-ay6l2huht7vmk2h"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-6fw5ylwbqeuyd4j"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jwsqammdbw33izmv4f4r"}},{"phid":"PHID-XACT-TASK-ngf3kluojsxgotw"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"phid":"PHID-XACT-TASK-xbkmd5uqyp3v4wg"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ampnqxdjkwqj7tb"},{"hovercardSpec":{"objectPHID":"PHID-USER-r5pzlcquei6f4hb53rfg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-nyz4e3itkdgffl4it2f6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-v5ivbe52v7kmx74l5s75"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-v5ivbe52v7kmx74l5s75"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-nyz4e3itkdgffl4it2f6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ivx4oe4mjp6scgb"},{"hovercardSpec":{"objectPHID":"PHID-USER-lqhywe57ljzomf36owq7"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ybearlyqae26w7cbuela"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-satqx4lmludmk3z"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-3iqkfk5jhlb7wx6"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-szbwank7exbaykt"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-d6cbjjacqkvgiof"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-o2vmqxokqnvgy6d"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-g2iw5hmoj3dq3ep"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-thp7per4p36co7i"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-p6ceqi3wypj2rah"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-gvlhz55735gmnd3"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-j3nrnbbddn5dluk"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-qc6kgpzhisc42o4"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-j7z5iulsyviaika"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-xw4eswntwg3y72e"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-hs3sn5vqh47npib"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-oyzgfu7twz6zsh3"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-eq2klzvcdkzfo6o"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-3hvgmjdb5dotjx7"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-kf2al552fyuavvu"},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"phid":"PHID-XACT-TASK-snzcn7ax54h3eeu"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-jjgexfc4gsxag63"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-jzbqgebx4w5rg2x"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-rmpj3yteb6mmhoa"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-mjp4rcw54odbulf"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-dsrhjy45wdsf2nl"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-jgblc7fkhxpca7x"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-osith37ji6vt7h2"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-idkqzixvkqjyurt"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-weuvbxbuxtrncns"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-sod7zofb72njzie"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-ugd2mb7yp7fmr67"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-v3bopzmjteckbsq"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-dppg77whjzmoidv"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-2d2mnluietd3sf5"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-nypkfqh5q7kjzmg"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-xzj3kyg2f2ysbu7"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-l27o6fhqepwjtoq"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-temuusuenfelq2u"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-nhehqh446i3c3mg"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-zvmpqwvosam2pht"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-vbfh5xca6rqn5mi"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-aake7t6l5ywzwd7"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-q2ctbrgus56aokp"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-r47uzlxg66shrxr"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-vfynfmtgiifgir5"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-f3jgefwy4mfid2o"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-sxu3ouobdoxpqil"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-lnah4od3uv4rxji","anchor":"9479197"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ai4cfod7q4tnd3q","anchor":"9481098"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fizenexhq5j6hwj","anchor":"9481322"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-wfg7etakjundamm\/","ref":"T355617#9481488"},[],{"anchor":"9481488"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_206\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_207\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wfg7etakjundamm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_208\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wfg7etakjundamm","anchor":"9481488"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2zqntalwcwf5fyh","anchor":"9482519"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-hn5kcx4vjaniu3d\/","ref":"T355617#9482761"},[],{"anchor":"9482761"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_218\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_219\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hn5kcx4vjaniu3d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_220\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_221\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Deployment Training Graduate","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-hn5kcx4vjaniu3d","anchor":"9482761"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ay6l2huht7vmk2h\/","ref":"T355617#9482787"},[],{"anchor":"9482787"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_229\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_230\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ay6l2huht7vmk2h\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_231\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_232\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ay6l2huht7vmk2h","anchor":"9482787"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-6fw5ylwbqeuyd4j\/","ref":"T355617#9483224"},[],{"anchor":"9483224"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_238\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6fw5ylwbqeuyd4j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_240\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6fw5ylwbqeuyd4j","anchor":"9483224"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-lfqg6ppm2xn5jvx","anchor":"9483225"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ngf3kluojsxgotw\/","ref":"T355617#9483282"},[],{"anchor":"9483282"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_250\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ngf3kluojsxgotw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Deployment Training Graduate","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ngf3kluojsxgotw","anchor":"9483282"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-xbkmd5uqyp3v4wg\/","ref":"T355617#9484282"},[],{"anchor":"9484282"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_261\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xbkmd5uqyp3v4wg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_263\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-xbkmd5uqyp3v4wg","anchor":"9484282"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ampnqxdjkwqj7tb\/","ref":"T355617#9485627"},[],{"anchor":"9485627"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_272\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ampnqxdjkwqj7tb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_274\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_275\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ampnqxdjkwqj7tb","anchor":"9485627"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mwpnvyyyeryh4ct","anchor":"9486083"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-oswywahxb44n2gm","anchor":"9487945"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-feaknxgh6mlm26k","anchor":"9487964"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ivx4oe4mjp6scgb\/","ref":"T355617#9489194"},[],{"anchor":"9489194"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_292\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ivx4oe4mjp6scgb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_294\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ivx4oe4mjp6scgb","anchor":"9489194"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5ivesyxzi5prdrj","anchor":"9489591"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-satqx4lmludmk3z\/","ref":"T355617#9489596"},[],{"anchor":"9489596"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_304\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-satqx4lmludmk3z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_306\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-satqx4lmludmk3z","anchor":"9489596"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-3iqkfk5jhlb7wx6\/","ref":"T355617#9489597"},[],{"anchor":"9489597"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_313\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3iqkfk5jhlb7wx6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_315\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3iqkfk5jhlb7wx6","anchor":"9489597"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-szbwank7exbaykt\/","ref":"T355617#9489598"},[],{"anchor":"9489598"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_322\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-szbwank7exbaykt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_324\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-szbwank7exbaykt","anchor":"9489598"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-d6cbjjacqkvgiof\/","ref":"T355617#9489599"},[],{"anchor":"9489599"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_331\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-d6cbjjacqkvgiof\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_333\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-d6cbjjacqkvgiof","anchor":"9489599"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-o2vmqxokqnvgy6d\/","ref":"T355617#9489600"},[],{"anchor":"9489600"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_340\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-o2vmqxokqnvgy6d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_342\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-o2vmqxokqnvgy6d","anchor":"9489600"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-g2iw5hmoj3dq3ep\/","ref":"T355617#9489601"},[],{"anchor":"9489601"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_349\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-g2iw5hmoj3dq3ep\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_351\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-g2iw5hmoj3dq3ep","anchor":"9489601"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-thp7per4p36co7i\/","ref":"T355617#9489634"},[],{"anchor":"9489634"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_358\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-thp7per4p36co7i\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_360\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_361\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-thp7per4p36co7i","anchor":"9489634"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-p6ceqi3wypj2rah\/","ref":"T355617#9489635"},[],{"anchor":"9489635"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_367\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-p6ceqi3wypj2rah\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_369\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_370\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-p6ceqi3wypj2rah","anchor":"9489635"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-gvlhz55735gmnd3\/","ref":"T355617#9490902"},[],{"anchor":"9490902"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_376\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gvlhz55735gmnd3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_378\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gvlhz55735gmnd3","anchor":"9490902"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-j3nrnbbddn5dluk\/","ref":"T355617#9490988"},[],{"anchor":"9490988"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_385\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j3nrnbbddn5dluk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_387\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-j3nrnbbddn5dluk","anchor":"9490988"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-qc6kgpzhisc42o4\/","ref":"T355617#9491010"},[],{"anchor":"9491010"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_394\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_395\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qc6kgpzhisc42o4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_396\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_397\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-qc6kgpzhisc42o4","anchor":"9491010"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-j7z5iulsyviaika\/","ref":"T355617#9491031"},[],{"anchor":"9491031"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_403\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j7z5iulsyviaika\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_405\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_406\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-j7z5iulsyviaika","anchor":"9491031"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-xw4eswntwg3y72e\/","ref":"T355617#9491088"},[],{"anchor":"9491088"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_412\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_413\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xw4eswntwg3y72e\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_414\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_415\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xw4eswntwg3y72e","anchor":"9491088"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-hs3sn5vqh47npib\/","ref":"T355617#9491108"},[],{"anchor":"9491108"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_421\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_422\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hs3sn5vqh47npib\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_423\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-hs3sn5vqh47npib","anchor":"9491108"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-oyzgfu7twz6zsh3\/","ref":"T355617#9491149"},[],{"anchor":"9491149"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_430\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-oyzgfu7twz6zsh3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_432\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-oyzgfu7twz6zsh3","anchor":"9491149"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-eq2klzvcdkzfo6o\/","ref":"T355617#9491150"},[],{"anchor":"9491150"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_439\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eq2klzvcdkzfo6o\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_441\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-eq2klzvcdkzfo6o","anchor":"9491150"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-3hvgmjdb5dotjx7\/","ref":"T355617#9491260"},[],{"anchor":"9491260"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_448\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3hvgmjdb5dotjx7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_450\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3hvgmjdb5dotjx7","anchor":"9491260"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-kf2al552fyuavvu\/","ref":"T355617#9491332"},[],{"anchor":"9491332"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_457\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kf2al552fyuavvu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_459\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_460\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kf2al552fyuavvu","anchor":"9491332"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-snzcn7ax54h3eeu\/","ref":"T355617#9493745"},[],{"anchor":"9493745"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_466\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_467\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-snzcn7ax54h3eeu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_468\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_469\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-snzcn7ax54h3eeu","anchor":"9493745"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-jjgexfc4gsxag63\/","ref":"T355617#9495837"},[],{"anchor":"9495837"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_475\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_476\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jjgexfc4gsxag63\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_477\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_478\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jjgexfc4gsxag63","anchor":"9495837"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-jzbqgebx4w5rg2x\/","ref":"T355617#9495846"},[],{"anchor":"9495846"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_484\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jzbqgebx4w5rg2x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_486\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_487\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jzbqgebx4w5rg2x","anchor":"9495846"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-rmpj3yteb6mmhoa\/","ref":"T355617#9495849"},[],{"anchor":"9495849"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_493\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_494\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rmpj3yteb6mmhoa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_495\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_496\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rmpj3yteb6mmhoa","anchor":"9495849"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-mjp4rcw54odbulf\/","ref":"T355617#9498724"},[],{"anchor":"9498724"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_502\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_503\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mjp4rcw54odbulf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_504\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_505\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mjp4rcw54odbulf","anchor":"9498724"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-dsrhjy45wdsf2nl\/","ref":"T355617#9498806"},[],{"anchor":"9498806"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_511\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_512\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dsrhjy45wdsf2nl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_513\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_514\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dsrhjy45wdsf2nl","anchor":"9498806"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-jgblc7fkhxpca7x\/","ref":"T355617#9498868"},[],{"anchor":"9498868"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_520\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_521\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jgblc7fkhxpca7x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_522\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_523\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jgblc7fkhxpca7x","anchor":"9498868"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-osith37ji6vt7h2\/","ref":"T355617#9498877"},[],{"anchor":"9498877"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_529\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_530\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-osith37ji6vt7h2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_531\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_532\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-osith37ji6vt7h2","anchor":"9498877"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-idkqzixvkqjyurt\/","ref":"T355617#9498879"},[],{"anchor":"9498879"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_538\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_539\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-idkqzixvkqjyurt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_540\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_541\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-idkqzixvkqjyurt","anchor":"9498879"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-weuvbxbuxtrncns\/","ref":"T355617#9498940"},[],{"anchor":"9498940"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_547\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_548\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-weuvbxbuxtrncns\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_549\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_550\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-weuvbxbuxtrncns","anchor":"9498940"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-sod7zofb72njzie\/","ref":"T355617#9499044"},[],{"anchor":"9499044"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_556\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_557\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sod7zofb72njzie\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_558\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_559\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-sod7zofb72njzie","anchor":"9499044"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ugd2mb7yp7fmr67\/","ref":"T355617#9499184"},[],{"anchor":"9499184"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_565\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ugd2mb7yp7fmr67\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_567\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_568\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ugd2mb7yp7fmr67","anchor":"9499184"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-v3bopzmjteckbsq\/","ref":"T355617#9499340"},[],{"anchor":"9499340"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_574\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_575\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v3bopzmjteckbsq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_576\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_577\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-v3bopzmjteckbsq","anchor":"9499340"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-dppg77whjzmoidv\/","ref":"T355617#9499827"},[],{"anchor":"9499827"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_583\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_584\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dppg77whjzmoidv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_585\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_586\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dppg77whjzmoidv","anchor":"9499827"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-2d2mnluietd3sf5\/","ref":"T355617#9499828"},[],{"anchor":"9499828"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_592\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_593\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2d2mnluietd3sf5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_594\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_595\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2d2mnluietd3sf5","anchor":"9499828"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-nypkfqh5q7kjzmg\/","ref":"T355617#9499890"},[],{"anchor":"9499890"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_601\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_602\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nypkfqh5q7kjzmg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_603\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_604\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-nypkfqh5q7kjzmg","anchor":"9499890"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-xzj3kyg2f2ysbu7\/","ref":"T355617#9499897"},[],{"anchor":"9499897"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_610\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_611\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xzj3kyg2f2ysbu7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_612\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_613\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xzj3kyg2f2ysbu7","anchor":"9499897"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-l27o6fhqepwjtoq\/","ref":"T355617#9500156"},[],{"anchor":"9500156"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_619\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_620\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l27o6fhqepwjtoq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_621\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_622\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-l27o6fhqepwjtoq","anchor":"9500156"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-temuusuenfelq2u\/","ref":"T355617#9500164"},[],{"anchor":"9500164"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_628\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_629\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-temuusuenfelq2u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_630\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_631\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-temuusuenfelq2u","anchor":"9500164"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-nhehqh446i3c3mg\/","ref":"T355617#9500192"},[],{"anchor":"9500192"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_637\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_638\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nhehqh446i3c3mg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_639\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_640\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-nhehqh446i3c3mg","anchor":"9500192"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-zvmpqwvosam2pht\/","ref":"T355617#9500274"},[],{"anchor":"9500274"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_646\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_647\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zvmpqwvosam2pht\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_648\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_649\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zvmpqwvosam2pht","anchor":"9500274"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-vbfh5xca6rqn5mi\/","ref":"T355617#9500310"},[],{"anchor":"9500310"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_655\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_656\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vbfh5xca6rqn5mi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_657\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_658\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vbfh5xca6rqn5mi","anchor":"9500310"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-aake7t6l5ywzwd7\/","ref":"T355617#9500403"},[],{"anchor":"9500403"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_664\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_665\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aake7t6l5ywzwd7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_666\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_667\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-aake7t6l5ywzwd7","anchor":"9500403"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-q2ctbrgus56aokp\/","ref":"T355617#9500404"},[],{"anchor":"9500404"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_673\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_674\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-q2ctbrgus56aokp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_675\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_676\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-q2ctbrgus56aokp","anchor":"9500404"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-r47uzlxg66shrxr\/","ref":"T355617#9500535"},[],{"anchor":"9500535"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_682\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_683\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r47uzlxg66shrxr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_684\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_685\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-r47uzlxg66shrxr","anchor":"9500535"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-vfynfmtgiifgir5\/","ref":"T355617#9500537"},[],{"anchor":"9500537"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_691\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_692\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vfynfmtgiifgir5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_693\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_694\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vfynfmtgiifgir5","anchor":"9500537"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-f3jgefwy4mfid2o\/","ref":"T355617#9502766"},[],{"anchor":"9502766"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_700\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_701\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-f3jgefwy4mfid2o\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_702\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_703\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-f3jgefwy4mfid2o","anchor":"9502766"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-sxu3ouobdoxpqil\/","ref":"T355617#9502848"},[],{"anchor":"9502848"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_709\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_710\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_215\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sxu3ouobdoxpqil\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_711\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_712\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-sxu3ouobdoxpqil","anchor":"9502848"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cc8a2c6b\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}