for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_119\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10026541\" id=\"10026541\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eVolans\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10026541\" data-sigil=\"has-tooltip\" data-meta=\"0_117\"\u003e\u003cspan class=\"screen-only\"\u003eJul 30 2024, 8:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-30 08:46:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10026556\" id=\"10026556\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_121\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eMarostegui\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10026556\" data-sigil=\"has-tooltip\" data-meta=\"0_120\"\u003e\u003cspan class=\"screen-only\"\u003eJul 30 2024, 8:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-30 08:48:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003eMarostegui\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eTriage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eReady\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eDBA\u003c\/a\u003e board.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_126\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kqrotiicoqdt2k4pm5ir\/PHID-FILE-v4ht3gubxnynd3i3oqdk\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Raine\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10028209\" id=\"10028209\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Raine\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eRaine\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10028209\" data-sigil=\"has-tooltip\" data-meta=\"0_124\"\u003e\u003cspan class=\"screen-only\"\u003eJul 30 2024, 2:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-30 14:56:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10028923\" id=\"10028923\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10028923\" data-sigil=\"has-tooltip\" data-meta=\"0_133\"\u003e\u003cspan class=\"screen-only\"\u003eJul 30 2024, 5:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-30 17:29:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_131\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1058225 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/software\/spicerack@master] mysql_legacy: instance improvements\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1058225\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1058225\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_138\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10028924\" id=\"10028924\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_137\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10028924\" data-sigil=\"has-tooltip\" data-meta=\"0_136\"\u003e\u003cspan class=\"screen-only\"\u003eJul 30 2024, 5:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-30 17:29:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10031763\" id=\"10031763\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eMarostegui\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eReady\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eIn progress\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eDBA\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10031763\" data-sigil=\"has-tooltip\" data-meta=\"0_139\"\u003e\u003cspan class=\"screen-only\"\u003eJul 31 2024, 12:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-31 12:48:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10032167\" id=\"10032167\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10032167\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"screen-only\"\u003eJul 31 2024, 2:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-31 14:43:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_146\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1058225 \u003cstrong\u003emerged\u003c\/strong\u003e by jenkins-bot:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/software\/spicerack@master] mysql_legacy: instance improvements\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1058225\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1058225\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10032417\" id=\"10032417\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10032417\" data-sigil=\"has-tooltip\" data-meta=\"0_151\"\u003e\u003cspan class=\"screen-only\"\u003eJul 31 2024, 3:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-07-31 15:30:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10035019\" id=\"10035019\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10035019\" data-sigil=\"has-tooltip\" data-meta=\"0_160\"\u003e\u003cspan class=\"screen-only\"\u003eAug 1 2024, 12:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-01 12:12:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_158\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1059052 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.switchdc.databases.preparation: new cookbooks\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1059052\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1059052\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10035020\" id=\"10035020\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_170\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10035020\" data-sigil=\"has-tooltip\" data-meta=\"0_169\"\u003e\u003cspan class=\"screen-only\"\u003eAug 1 2024, 12:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-01 12:12:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_167\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1059053 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cookbooks: add config for sre.switchdc.databases\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1059053\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1059053\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10035961\" id=\"10035961\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10035961\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"screen-only\"\u003eAug 1 2024, 3:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-01 15:53:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_176\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1059053 \u003cstrong\u003emerged\u003c\/strong\u003e by Volans:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cookbooks: add config for sre.switchdc.databases\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1059053\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1059053\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_193\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" aria-label=\"Nerd Sniper\" data-sigil=\"has-tooltip\" data-meta=\"0_189\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_190\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" aria-label=\"Rat Catcher\" data-sigil=\"has-tooltip\" data-meta=\"0_191\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_192\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10035997\" id=\"10035997\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_188\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10035997\" data-sigil=\"has-tooltip\" data-meta=\"0_187\"\u003e\u003cspan class=\"screen-only\"\u003eAug 1 2024, 4:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-01 16:00:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_185\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_52\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eStatus update:\u003cbr \/\u003e\nSome addition to spicerack were made in this \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/c\/operations\/software\/spicerack\/+\/1058225\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003epatch\u003c\/a\u003e. I would actually be tempted to rename \u003ctt class=\"remarkup-monospaced\"\u003eInstance.run_vertical_query()\u003c\/tt\u003e to \u003ctt class=\"remarkup-monospaced\"\u003eInstance.run_query()\u003c\/tt\u003e and replace it as it's the only way to sensibly parse some output (with the caveats explained in the docstring).\u003cbr \/\u003e\nWith the \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1059053\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003epuppet patch\u003c\/a\u003e we save the replication credentials to the cumin hosts in a way accessibile by the cookbooks it will be possible to start testing (with \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Spicerack\/Cookbooks#Test_before_merging\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003etest-cookbook\u003c\/a\u003e \u003cstrong\u003epassing the DRY-RUN mode option\u003c\/strong\u003e) the two new cookbooks introduced as good draft but still WIP \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/c\/operations\/cookbooks\/+\/1059052\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehere\u003c\/a\u003e for the pre and post steps required for the switch datacenter.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_206\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" aria-label=\"Nerd Sniper\" data-sigil=\"has-tooltip\" data-meta=\"0_202\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" aria-label=\"Rat Catcher\" data-sigil=\"has-tooltip\" data-meta=\"0_204\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_205\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10036520\" id=\"10036520\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003eVolans\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eMarostegui\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10036520\" data-sigil=\"has-tooltip\" data-meta=\"0_200\"\u003e\u003cspan class=\"screen-only\"\u003eAug 1 2024, 6:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-01 18:40:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_198\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_199\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFinal update before vacations :)\u003c\/p\u003e\n\n\u003cp\u003eI've update the above patch and performed a DRY-RUN test on both cookbooks forcing to run on only one section (\u003ctt class=\"remarkup-monospaced\"\u003ex1\u003c\/tt\u003e):\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.prepare -t AAA eqiad codfw\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.finalize -t AAA codfw eqiad\u003c\/tt\u003e  This is with DC inverted as otherwise would not pass the initial validation\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e you can inspect the logs of the runs on \u003ctt class=\"remarkup-monospaced\"\u003ecumin2002:\/home\/volans\/cookbooks_testing\/logs\/sre\/switchdc\/databases\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cp\u003eYou would not find the \u003ctt class=\"remarkup-monospaced\"\u003eCHANGE MASTER\u003c\/tt\u003e query in the logs due to the hack to avoid leaking to the logs the replication password (to be properly fixed when I'm back). Here's the actually run query:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eExecuting commands ['\/usr\/local\/bin\/mysql --socket \/run\/mysqld\/mysqld.sock --batch --execute "CHANGE MASTER TO master_host=\\'db2196.codfw.wmnet\\',\\nmaster_port=3306,\\nmaster_ssl=1,\\nmaster_log_file=\\'db2196-bin.002532\\',\\nmaster_log_pos=347987607,\\nmaster_user=\\'repl2024\\',\\nmaster_password=\\'REDACTED\\'"'] on 1 hosts: db1220.eqiad.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eHere below are the summaries (there will be one per section) that will be saved into Phabricator (can be fine-tuned in the formatting ofc, probably shortening the FQDNs to hostnames and making \u003ctt class=\"remarkup-monospaced\"\u003emonospace\u003c\/tt\u003e various bits, TBD :D ).\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item phantom-item\"\u003e\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003esre.switchdc.databases.prepare\u003c\/tt\u003e\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ex1 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section x1 between MASTER_FROM db1220.eqiad.wmnet and MASTER_TO db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet MASTER_USE_GTID=no.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet Ignoring wrong SLAVE STATUS in DRY-RUN mode Using_Gtid=Slave_Pos, expected no instead.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled GTID on MASTER_TO db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet stopped pt-heartbeat.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet Ignoring MASTER STATUS is not stable in DRY-RUN\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db1220.eqiad.wmnet CHANGE MASTER to ReplicationInfo(primary='db2196.codfw.wmnet', binlog='db2196-bin.002533', position=394662451, port=3306) and user repl2024\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db1220.eqiad.wmnet START SLAVE\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db1220.eqiad.wmnet skipping replication from MASTER_TO db2196.codfw.wmnet verification\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet started pt-heartbeat.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet replication from MASTER_FROM db1220.eqiad.wmnet verified\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003ch6 class=\"remarkup-header\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003esre.switchdc.databases.finalize\u003c\/tt\u003e\u003c\/h6\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ex1 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section x1 between MASTER_TO db1220.eqiad.wmnet and MASTER_FROM db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db1220.eqiad.wmnet has no replication set, skipping.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet heartbeat server IDs to delete are: []\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet MASTER_USE_GTID=slave_pos.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIgnoring failed check for GTID change in DRY-RUN mode.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEnabled GTID on MASTER_FROM db2196.codfw.wmnet\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThe procedure did not test 100% of the code (like there are no rows to delete in the heartbeat table), but I guess at some point we could do some live test of it or in a controlled environment.\u003cbr \/\u003e\n\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e if you could look at the code and the logs to validate the procedure is sound and does what you want before your sabbatical that would be great!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_219\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_215\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" aria-label=\"Nerd Sniper\" data-sigil=\"has-tooltip\" data-meta=\"0_217\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10063599\" id=\"10063599\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_214\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eMarostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10063599\" data-sigil=\"has-tooltip\" data-meta=\"0_213\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2024, 8:55 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-14 08:55:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_211\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T371351#10036520\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT371351#10036520\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Volans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Volans\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eFinal update before vacations :)\u003c\/p\u003e\n\n\u003cp\u003eI've update the above patch and performed a DRY-RUN test on both cookbooks forcing to run on only one section (\u003ctt class=\"remarkup-monospaced\"\u003ex1\u003c\/tt\u003e):\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.prepare -t AAA eqiad codfw\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.finalize -t AAA codfw eqiad\u003c\/tt\u003e  This is with DC inverted as otherwise would not pass the initial validation\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e you can inspect the logs of the runs on \u003ctt class=\"remarkup-monospaced\"\u003ecumin2002:\/home\/volans\/cookbooks_testing\/logs\/sre\/switchdc\/databases\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cp\u003eYou would not find the \u003ctt class=\"remarkup-monospaced\"\u003eCHANGE MASTER\u003c\/tt\u003e query in the logs due to the hack to avoid leaking to the logs the replication password (to be properly fixed when I'm back). Here's the actually run query:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eExecuting commands ['\/usr\/local\/bin\/mysql --socket \/run\/mysqld\/mysqld.sock --batch --execute "CHANGE MASTER TO master_host=\\'db2196.codfw.wmnet\\',\\nmaster_port=3306,\\nmaster_ssl=1,\\nmaster_log_file=\\'db2196-bin.002532\\',\\nmaster_log_pos=347987607,\\nmaster_user=\\'repl2024\\',\\nmaster_password=\\'REDACTED\\'"'] on 1 hosts: db1220.eqiad.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eHere below are the summaries (there will be one per section) that will be saved into Phabricator (can be fine-tuned in the formatting ofc, probably shortening the FQDNs to hostnames and making \u003ctt class=\"remarkup-monospaced\"\u003emonospace\u003c\/tt\u003e various bits, TBD :D ).\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item phantom-item\"\u003e\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003esre.switchdc.databases.prepare\u003c\/tt\u003e\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ex1 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section x1 between MASTER_FROM db1220.eqiad.wmnet and MASTER_TO db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet MASTER_USE_GTID=no.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet Ignoring wrong SLAVE STATUS in DRY-RUN mode Using_Gtid=Slave_Pos, expected no instead.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled GTID on MASTER_TO db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet stopped pt-heartbeat.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet Ignoring MASTER STATUS is not stable in DRY-RUN\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db1220.eqiad.wmnet CHANGE MASTER to ReplicationInfo(primary='db2196.codfw.wmnet', binlog='db2196-bin.002533', position=394662451, port=3306) and user repl2024\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db1220.eqiad.wmnet START SLAVE\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db1220.eqiad.wmnet skipping replication from MASTER_TO db2196.codfw.wmnet verification\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet started pt-heartbeat.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet replication from MASTER_FROM db1220.eqiad.wmnet verified\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI haven't checked the code but the order of the steps look good to me to prepare before the switch.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003ch6 class=\"remarkup-header\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003esre.switchdc.databases.finalize\u003c\/tt\u003e\u003c\/h6\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ex1 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section x1 between MASTER_TO db1220.eqiad.wmnet and MASTER_FROM db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db1220.eqiad.wmnet has no replication set, skipping.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet heartbeat server IDs to delete are: []\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet MASTER_USE_GTID=slave_pos.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIgnoring failed check for GTID change in DRY-RUN mode.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEnabled GTID on MASTER_FROM db2196.codfw.wmnet\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI need to check the code in more details to see what this finalize step because there are some things that I don't get.\u003cbr \/\u003e\nIf this is the cookbook to run once the DC switchover has finished (so a day after it - as the code says) the order doesn't seem correct\u003c\/p\u003e\n\n\u003cp\u003eIf we are switching from eqiad to codfw that means that after the switch codfw is primary and eqiad is secondary. \u003cbr \/\u003e\nAfter the switch, db1120 would have replication enabled, and the only needed thing would be to:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003edb2196, remove circular replication so: stop slave; reset slave all; so no need to enable GTID there. Primary masters do not replicate from anyone\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003edb1120 (now a normal slave), enable GTID (it should have replication enabled already, coming from the \u003ctt class=\"remarkup-monospaced\"\u003eprepare\u003c\/tt\u003e step)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_222\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wk46ogi4j5zqtxoxny5r\/PHID-FILE-27qx47g5qvmxrjmrqqpp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ABran-WMF\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10077492\" id=\"10077492\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_221\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ABran-WMF\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eABran-WMF\u003c\/a\u003e mentioned this in \u003ca href=\"\/T372893\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003eT372893: Testing environment for mysql cookbooks\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10077492\" data-sigil=\"has-tooltip\" data-meta=\"0_220\"\u003e\u003cspan class=\"screen-only\"\u003eAug 20 2024, 1:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-20 13:39:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_225\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wk46ogi4j5zqtxoxny5r\/PHID-FILE-27qx47g5qvmxrjmrqqpp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ABran-WMF\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10080901\" id=\"10080901\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_224\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ABran-WMF\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003eABran-WMF\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/data-persistence-automations\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003eData-Persistence-Automations\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10080901\" data-sigil=\"has-tooltip\" data-meta=\"0_223\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2024, 11:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-21 11:19:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_234\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wk46ogi4j5zqtxoxny5r\/PHID-FILE-27qx47g5qvmxrjmrqqpp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ABran-WMF\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10106203\" id=\"10106203\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ABran-WMF\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003eABran-WMF\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10106203\" data-sigil=\"has-tooltip\" data-meta=\"0_232\"\u003e\u003cspan class=\"screen-only\"\u003eAug 30 2024, 12:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-08-30 12:51:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_230\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAfter testing cookbooks in \u003ca href=\"\/T372893\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT372893\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e:\u003cbr \/\u003e\n\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/c\/operations\/cookbooks\/+\/1059052\/comment\/ec140e0a_377b3e9f\/\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003espotted\u003c\/a\u003e a SQL syntax issue which I fixed, but it broke CI somehow. and  I've spotted a micro case sensitivity issue on \u003ca href=\"https:\/\/github.com\/wikimedia\/operations-software-spicerack\/blob\/28b6f7c5dcd295c2f6351af6a958022ea45effa5\/spicerack\/mysql_legacy.py#L49-L62\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003eMasterUseGTID\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T372893#10102685\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT372893#10102685\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ABran-WMF\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ABran-WMF\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T372893#10102518\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT372893#10102518\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ops-monitoring-bot\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003ecookbooks.sre.switchdc.databases for the switch from test-s4 to test-s4 started by arnaudb@cumin1002 completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2230.codfw.wmnet MASTER_USE_GTID=slave_pos.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eFailed to enable GTID on db2230.codfw.wmnet, current value: Slave_Pos\u003c\/strong\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eit appears that\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T372893#10102444\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT372893#10102444\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ABran-WMF\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ABran-WMF\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI've spotted slight issue on \u003ctt class=\"remarkup-monospaced\"\u003eclass MasterUseGTID(Enum):\u003c\/tt\u003e: [...]\u003ctt class=\"remarkup-monospaced\"\u003eNO = "no"\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eapplies as well on that value\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Volans\u003c\/span\u003e\u003c\/a\u003e: Testing platform is still up and running so we can try and tie the remaining few loose ends, lmk if and how I can help!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_247\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" aria-label=\"Nerd Sniper\" data-sigil=\"has-tooltip\" data-meta=\"0_243\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_244\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" aria-label=\"Rat Catcher\" data-sigil=\"has-tooltip\" data-meta=\"0_245\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10110285\" id=\"10110285\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10110285\" data-sigil=\"has-tooltip\" data-meta=\"0_241\"\u003e\u003cspan class=\"screen-only\"\u003eSep 2 2024, 1:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-02 13:09:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_239\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T371351#10063599\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT371351#10063599\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003ch6 class=\"remarkup-header\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003esre.switchdc.databases.finalize\u003c\/tt\u003e\u003c\/h6\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ex1 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section x1 between MASTER_TO db1220.eqiad.wmnet and MASTER_FROM db2196.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db1220.eqiad.wmnet has no replication set, skipping.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2196.codfw.wmnet heartbeat server IDs to delete are: []\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet MASTER_USE_GTID=slave_pos.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM db2196.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIgnoring failed check for GTID change in DRY-RUN mode.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEnabled GTID on MASTER_FROM db2196.codfw.wmnet\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI need to check the code in more details to see what this finalize step because there are some things that I don't get.\u003cbr \/\u003e\nIf this is the cookbook to run once the DC switchover has finished (so a day after it - as the code says) the order doesn't seem correct\u003c\/p\u003e\n\n\u003cp\u003eIf we are switching from eqiad to codfw that means that after the switch codfw is primary and eqiad is secondary. \u003cbr \/\u003e\nAfter the switch, db1120 would have replication enabled, and the only needed thing would be to:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003edb2196, remove circular replication so: stop slave; reset slave all; so no need to enable GTID there. Primary masters do not replicate from anyone\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003edb1120 (now a normal slave), enable GTID (it should have replication enabled already, coming from the \u003ctt class=\"remarkup-monospaced\"\u003eprepare\u003c\/tt\u003e step)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eAs I mentioned in my previous message, the \u003ctt class=\"remarkup-monospaced\"\u003efinalize\u003c\/tt\u003e one is run inverted:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.finalize -t AAA codfw eqiad\u003c\/tt\u003e This is with DC inverted as otherwise would not pass the initial validation\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_250\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wk46ogi4j5zqtxoxny5r\/PHID-FILE-27qx47g5qvmxrjmrqqpp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ABran-WMF\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10112272\" id=\"10112272\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ABran-WMF\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003eABran-WMF\u003c\/a\u003e mentioned this in \u003ca href=\"\/T373579\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003eT373579: Productionize db22[21-40]\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10112272\" data-sigil=\"has-tooltip\" data-meta=\"0_248\"\u003e\u003cspan class=\"screen-only\"\u003eSep 3 2024, 10:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-03 10:19:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_253\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10152628\" id=\"10152628\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_252\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003ejcrespo\u003c\/a\u003e added a subtask: \u003ca href=\"\/T374933\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eT374933: Add section alias for databases in the test-s1 and test-s4 sections\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10152628\" data-sigil=\"has-tooltip\" data-meta=\"0_251\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 11:35 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 11:35:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_265\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_263\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153483\" id=\"10153483\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003ejcrespo\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#10153483\" data-sigil=\"has-tooltip\" data-meta=\"0_261\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:29:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_259\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_70\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI executed:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.prepare -t T371351 eqiad codfw\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eand got:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eDRY-RUN: Validated replication topology for section test-s4 between MASTER_FROM db1125.eqiad.wmnet and MASTER_TO db2230.codfw.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003elater, I wanted to test this erroring, so I did:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.prepare -t T371351 codfw eqiad\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e(note the wrong dc direction)\u003c\/p\u003e\n\n\u003cp\u003eBut it validated correctly:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eDRY-RUN: Validated replication topology for section test-s4 between MASTER_FROM db1125.eqiad.wmnet and MASTER_TO db2230.codfw.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eShouldn't it fail, because I asked to switchover in the wrong direction, and it should fail and warn rather than silently autocorrect? As otherwise, if for some reason replication was flowing in the wrong direction, it would just add the other link, rather than warn and let the operator either continue, fix or abort.\u003c\/p\u003e\n\n\u003cp\u003eI was able to make it fail when setting up circual replication, as it did:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eDRY-RUN: Failed to run cookbooks.sre.switchdc.databases.prepare.PrepareSection.validate: Expected all replicas of MASTER_TO db2230.codfw.wmnet to be in the same datacenter, got db1125.eqiad.wmnet instead\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI think this is ok, although I would alter slightly the error message\/add a dedicated check to not touch the topology if it was already circular replication.\u003c\/p\u003e\n\n\u003cp\u003eThis is not a blocker, but noting everything weird I see. I will keep testing the rest.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_274\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153571\" id=\"10153571\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10153571\" data-sigil=\"has-tooltip\" data-meta=\"0_272\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:45:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_270\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.switchdc.databases for the switch from eqiad to codfw started by jynus@cumin1002\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_283\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153573\" id=\"10153573\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10153573\" data-sigil=\"has-tooltip\" data-meta=\"0_281\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:46:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_279\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.switchdc.databases for the switch from eqiad to codfw started by jynus@cumin1002 completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003etest-s4 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section test-s4 between MASTER_FROM db1125.eqiad.wmnet and MASTER_TO db2230.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2230.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2230.codfw.wmnet MASTER_USE_GTID=no.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2230.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled GTID on MASTER_TO db2230.codfw.wmnet\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_292\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153579\" id=\"10153579\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10153579\" data-sigil=\"has-tooltip\" data-meta=\"0_290\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:47:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_288\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.switchdc.databases for the switch from eqiad to codfw started by jynus@cumin1002\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_301\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/drt7mnx6d3jikcn25k46\/PHID-FILE-pispkttunumodg6i4ghv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ops-monitoring-bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153582\" id=\"10153582\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ops-monitoring-bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003eops-monitoring-bot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10153582\" data-sigil=\"has-tooltip\" data-meta=\"0_299\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:47:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_297\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecookbooks.sre.switchdc.databases for the switch from eqiad to codfw started by jynus@cumin1002 completed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003etest-s4 (\u003cstrong\u003ePASS\u003c\/strong\u003e)\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eValidated replication topology for section test-s4 between MASTER_FROM db1125.eqiad.wmnet and MASTER_TO db2230.codfw.wmnet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2230.codfw.wmnet STOP SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2230.codfw.wmnet MASTER_USE_GTID=no.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO db2230.codfw.wmnet START SLAVE.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisabled GTID on MASTER_TO db2230.codfw.wmnet\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_313\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_311\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153584\" id=\"10153584\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#10153584\" data-sigil=\"has-tooltip\" data-meta=\"0_309\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:49:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_307\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI am going to create a dedicated task for production testing, to avoid also noise here and on IRC: \u003ca href=\"\/T374972\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT374972\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_324\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_322\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153616\" id=\"10153616\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10153616\" data-sigil=\"has-tooltip\" data-meta=\"0_320\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 14:57:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_318\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe other thing I saw after \u003ca href=\"\/T371351#10153483\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT371351#10153483\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e is that on the next step, if I run twice the disabling of GTID, there is no error or warning.\u003c\/p\u003e\n\n\u003cp\u003eIn this case, I think it is ok to not be a fatal error, as it would be ok to disable gtid and even leave it disabled for some time (it is not a huge error), but I would like to see at least \u003cstrong\u003ea warning\u003c\/strong\u003e that it was disabled in the first place (of if execution aborted before, but not ok if it was disabled for a long time). I think this should be a check to do beforehand (even if not an error).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_337\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" aria-label=\"Nerd Sniper\" data-sigil=\"has-tooltip\" data-meta=\"0_333\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" aria-label=\"Rat Catcher\" data-sigil=\"has-tooltip\" data-meta=\"0_335\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153650\" id=\"10153650\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10153650\" data-sigil=\"has-tooltip\" data-meta=\"0_331\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 3:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 15:03:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_329\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T371351#10153483\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT371351#10153483\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI executed:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.prepare -t T371351 eqiad codfw\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eand got:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eDRY-RUN: Validated replication topology for section test-s4 between MASTER_FROM db1125.eqiad.wmnet and MASTER_TO db2230.codfw.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003elater, I wanted to test this erroring, so I did:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003etest-cookbook -c 1059052 --dry-run sre.switchdc.databases.prepare -t T371351 codfw eqiad\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e(note the wrong dc direction)\u003c\/p\u003e\n\n\u003cp\u003eBut it validated correctly:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eDRY-RUN: Validated replication topology for section test-s4 between MASTER_FROM db1125.eqiad.wmnet and MASTER_TO db2230.codfw.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eShouldn't it fail, because I asked to switchover in the wrong direction, and it should fail and warn rather than silently autocorrect? As otherwise, if for some reason replication was flowing in the wrong direction, it would just add the other link, rather than warn and let the operator either continue, fix or abort.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e that's expected due to the patch to make it run for test-s4, we bypassed the input DCs, so dc_from and dc_to are not really used as we have forced master_from and master_to to be the specific ones.\u003c\/p\u003e\n\n\u003cp\u003eFYI those are the tests performed:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_FROM is RW\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMASTER_TO is  RO\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ethe replication master of MASTER_TO is MASTER_FROM\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eall the hosts replicating from MASTER_TO are in the same DC of MASTER_TO\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eIs there anything else I could add as validation step for the \u003ctt class=\"remarkup-monospaced\"\u003eprepare\u003c\/tt\u003e cookbook?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_349\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_347\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_348\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10153667\" id=\"10153667\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#10153667\" data-sigil=\"has-tooltip\" data-meta=\"0_345\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 15:07:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_343\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_344\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T371351#10153650\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT371351#10153650\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Volans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Volans\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThank you, I hadn't thought of that, it makes sense.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIs there anything else I could add as validation step for the \u003ctt class=\"remarkup-monospaced\"\u003eprepare\u003c\/tt\u003e cookbook?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSee \u003ca href=\"\/T371351#10153616\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT371351#10153616\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e. Although please note, I am using this ticket as my notes step-by-step, and given the time not all may be necessary\/big blocker. Will summarize it at the end when I have gone over multiple iterations\/failure scenarios.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_360\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_358\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10154105\" id=\"10154105\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_357\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10154105\" data-sigil=\"has-tooltip\" data-meta=\"0_356\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 4:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 16:24:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_354\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_355\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eone additional comment about the process, not necessarily the script, is that the post-maintenance script is confusing, as it will be ran post-maintenance, but the parameters will be in the direction of the maintenance (but replication will be flowing in the previous direction).\u003c\/p\u003e\n\n\u003cp\u003eWhile the topology check will take care of that, I wonder if there could be a way to prevent the confusion.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_373\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" aria-label=\"Nerd Sniper\" data-sigil=\"has-tooltip\" data-meta=\"0_369\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_370\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" aria-label=\"Rat Catcher\" data-sigil=\"has-tooltip\" data-meta=\"0_371\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_372\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10154152\" id=\"10154152\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10154152\" data-sigil=\"has-tooltip\" data-meta=\"0_367\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 4:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 16:32:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_365\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_366\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe idea was that this is all part of the A->B datacenter switchover and the finalize steps are still part of that process hence to be called with the same flow A->B. If you prefer otherwise we can invert it and\/or improve the help messages.\u003c\/p\u003e\n\n\u003cp\u003eAnother totally alternative approach could be the one to check which one is the primary DC for mediawiki and get the DC names from there automatically without having the user to pass them.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_384\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" aria-label=\"Mentor\" data-sigil=\"has-tooltip\" data-meta=\"0_382\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10154179\" id=\"10154179\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10154179\" data-sigil=\"has-tooltip\" data-meta=\"0_380\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2024, 4:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-17 16:38:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_378\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI found an actual bug: this is failing:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eFailed to run cookbooks.sre.switchdc.databases.finalize.FinalizeSection.clean_heartbeat: Failed to run 'DELETE FROM heartbeat WHERE server_id=180360463' on db1125.eqiad.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eEither:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eit returns it is failing but it is not (e.g. it deletes 0 rows) [confirmed it is not the case]\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eit is not running it on the right db (heartbeat) or not using heartbeat.heartbeat as a full qualifier\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eit is using a user without super user permisions [confirmed it is not the case, as read only is 0 here, but could be another persmission issue]\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eMy guess would be #2\u003c\/p\u003e\n\n\u003cp\u003eRe:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eThe idea was that this is all part of the A->B datacenter switchover and the finalize steps are still part of that proces\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI think I would leave it as is, but I would add somewhere noticeable "TO_DC IS EXPECTED TO BE THE PRIMARY" or something like that.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_393\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10156301\" id=\"10156301\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156301\" data-sigil=\"has-tooltip\" data-meta=\"0_391\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 9:15 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 09:15:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_389\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1059052 \u003cstrong\u003emerged\u003c\/strong\u003e by jenkins-bot:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.switchdc.databases: new cookbooks\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1059052\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1059052\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_396\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10156341\" id=\"10156341\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_395\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156341\" data-sigil=\"has-tooltip\" data-meta=\"0_394\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 9:31 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 09:31:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_405\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10156417\" id=\"10156417\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156417\" data-sigil=\"has-tooltip\" data-meta=\"0_403\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 10:02 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 10:02:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_401\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_402\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1073750 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] re.switchdc.databases.prepare: reduce wait time\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1073750\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1073750\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_408\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10156419\" id=\"10156419\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_407\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156419\" data-sigil=\"has-tooltip\" data-meta=\"0_406\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 10:02 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 10:02:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_417\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10156530\" id=\"10156530\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_416\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156530\" data-sigil=\"has-tooltip\" data-meta=\"0_415\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 11:04 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 11:04:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_413\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_414\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1073750 \u003cstrong\u003emerged\u003c\/strong\u003e by jenkins-bot:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] re.switchdc.databases.prepare: reduce wait time\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1073750\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1073750\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_426\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10156546\" id=\"10156546\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_425\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156546\" data-sigil=\"has-tooltip\" data-meta=\"0_424\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 11:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 11:09:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_422\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_423\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1073757 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.switchdc.databases: fix Phabricator message\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1073757\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1073757\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_435\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10156630\" id=\"10156630\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_434\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156630\" data-sigil=\"has-tooltip\" data-meta=\"0_433\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 11:39 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 11:39:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_431\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_432\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1073757 \u003cstrong\u003emerged\u003c\/strong\u003e by jenkins-bot:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.switchdc.databases: fix Phabricator message\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1073757\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1073757\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_438\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10156822\" id=\"10156822\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10156822\" data-sigil=\"has-tooltip\" data-meta=\"0_436\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 12:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 12:31:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_442\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"10158996\" id=\"10158996\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003ejcrespo\u003c\/a\u003e closed subtask \u003ca href=\"\/T374972\" class=\"phui-handle handle-status-closed\"\u003eT374972: Output test logs of production testing of the pre switchover tasks related to databases\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10158996\" data-sigil=\"has-tooltip\" data-meta=\"0_439\"\u003e\u003cspan class=\"screen-only\"\u003eSep 18 2024, 9:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-18 21:22:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_441\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003ejcrespo\u003c\/a\u003e reopened subtask \u003ca href=\"\/T374972\" class=\"phui-handle handle-status-closed\"\u003eT374972: Output test logs of production testing of the pre switchover tasks related to databases\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eIn Progress\u003c\/span\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_451\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10159982\" id=\"10159982\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_450\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10159982\" data-sigil=\"has-tooltip\" data-meta=\"0_449\"\u003e\u003cspan class=\"screen-only\"\u003eSep 19 2024, 10:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-19 10:08:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_447\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_448\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_113\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1074127 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.switchdc.databases.prepare: add check\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1074127\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1074127\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_460\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"10159983\" id=\"10159983\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_459\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_115\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#10159983\" data-sigil=\"has-tooltip\" data-meta=\"0_458\"\u003e\u003cspan class=\"screen-only\"\u003eSep 19 2024, 10:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2024-09-19 10:08:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_456\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_457\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_116\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange #1074128 had a related patch set uploaded (by Volans; author: Volans):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.switchdc.databases: update Phabricator more\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/1074128\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/1074128\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkqaqfsk7muk3ovm5v55"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkqaqfsk7muk3ovm5v55"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkqaqfsk7muk3ovm5v55"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkqaqfsk7muk3ovm5v55"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-oprgn52aabxc3rnzcmuu","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-oprgn52aabxc3rnzcmuu","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3ezvr3v65xkhst64byyf"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2","contextPHID":"PHID-TASK-jkyxt7diz5kjtzofzzuy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bk6sgdzdhctxupi3vn5b"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-3vdprvevg2kt3svyzu5e"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ws6yklybhvd6dsomvboy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-awvpp6ntxewc3kh"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-3vdprvevg2kt3svyzu5e"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-5iuvn4kjilwynldgts56"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-2t3qcpz7peyyfos"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-uggnpbp7wxtecbw"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"phid":"PHID-XACT-TASK-7734fgonly4ual2"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-f2ubqxuuclzczmj"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-wwmk32rpitu4s2z"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-rtlylknsunldqvy"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-ka4cpkmqiyl5b6i"},{"hovercardSpec":{"objectPHID":"PHID-USER-oprgn52aabxc3rnzcmuu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jkqaqfsk7muk3ovm5v55"}},{"hovercardSpec":{"objectPHID":"PHID-USER-oprgn52aabxc3rnzcmuu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-c737t63d3ari7ftysuo3"}},{"hovercardSpec":{"objectPHID":"PHID-USER-oprgn52aabxc3rnzcmuu"}},{"phid":"PHID-XACT-TASK-z6adowh7hjckih6"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-bwslk57zvhyy7d7"},{"hovercardSpec":{"objectPHID":"PHID-USER-oprgn52aabxc3rnzcmuu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vnufxs6gxcekjqegmfag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yh52wpo52yclnjxuieix"}},{"phid":"PHID-XACT-TASK-ugjlicxh6kijuz7"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-42cr2vrultubwwe"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-fuavtjkb3klv6df"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-wk67himigfcf6ny"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzxwjo5i7iwtojwoibwt"}},{"phid":"PHID-XACT-TASK-xialu76vfqxiiyy"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-yry4fdatsph5j4x"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-l34yj2a5isg56sa"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-pm6w3idracsanco"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-tde3rlclnp66xva"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-rw2ybqn2mm2jirw"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-dm65kygkulu4aeo"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-nua4vqurbqtz2hf"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-kcbzccbpg7b6vmk"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-bxomttsu4mskq6y"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-wevkle75knat4t4"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-cnvppia7cyna4t4"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-f4ww6oomj3q3gwb"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-q4hegfzzjaeixtm"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"phid":"PHID-XACT-TASK-apk56htzjbd2fo4"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-clhhpql722ca44r","anchor":"10026541"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-azldzcnduc3bbvl","anchor":"10026556"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sfo6ynatjbhiugg","anchor":"10028209"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-awvpp6ntxewc3kh\/","ref":"T371351#10028923"},[],{"anchor":"10028923"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_127\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_128\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-awvpp6ntxewc3kh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_129\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-awvpp6ntxewc3kh","anchor":"10028923"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5otnrapfp6bwt5p","anchor":"10028924"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5sfbvuko6raapyu","anchor":"10031763"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2t3qcpz7peyyfos\/","ref":"T371351#10032167"},[],{"anchor":"10032167"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_142\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2t3qcpz7peyyfos\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2t3qcpz7peyyfos","anchor":"10032167"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-l2flhnlkuizk4m7","anchor":"10032417"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uggnpbp7wxtecbw\/","ref":"T371351#10035019"},[],{"anchor":"10035019"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_154\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uggnpbp7wxtecbw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_156\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_157\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uggnpbp7wxtecbw","anchor":"10035019"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7734fgonly4ual2\/","ref":"T371351#10035020"},[],{"anchor":"10035020"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_163\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7734fgonly4ual2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_165\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-w3zjagenporfllf","anchor":"10035020"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-f2ubqxuuclzczmj\/","ref":"T371351#10035961"},[],{"anchor":"10035961"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-f2ubqxuuclzczmj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_174\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-f2ubqxuuclzczmj","anchor":"10035961"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wwmk32rpitu4s2z\/","ref":"T371351#10035997"},[],{"anchor":"10035997"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_181\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wwmk32rpitu4s2z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_183\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-wwmk32rpitu4s2z","anchor":"10035997"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rtlylknsunldqvy\/","ref":"T371351#10036520"},[],{"anchor":"10036520"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_194\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rtlylknsunldqvy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_196\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-d5v6y5mmjylzaoa","anchor":"10036520"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ka4cpkmqiyl5b6i\/","ref":"T371351#10063599"},[],{"anchor":"10063599"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_207\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_208\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ka4cpkmqiyl5b6i\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_209\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ka4cpkmqiyl5b6i","anchor":"10063599"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ghoh7zo6hg5dfqz","anchor":"10077492"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-65v77vdl4j33yhs","anchor":"10080901"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-z6adowh7hjckih6\/","ref":"T371351#10106203"},[],{"anchor":"10106203"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-z6adowh7hjckih6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_228\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z6adowh7hjckih6","anchor":"10106203"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bwslk57zvhyy7d7\/","ref":"T371351#10110285"},[],{"anchor":"10110285"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_235\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bwslk57zvhyy7d7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_237\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_238\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-bwslk57zvhyy7d7","anchor":"10110285"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-okxueyfoivtqhor","anchor":"10112272"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-f5ait6vvumtb57q","anchor":"10152628"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ugjlicxh6kijuz7\/","ref":"T371351#10153483"},[],{"anchor":"10153483"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_254\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ugjlicxh6kijuz7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_256\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-ugjlicxh6kijuz7\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_258\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ugjlicxh6kijuz7","anchor":"10153483"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-42cr2vrultubwwe\/","ref":"T371351#10153571"},[],{"anchor":"10153571"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_266\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-42cr2vrultubwwe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_268\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-42cr2vrultubwwe","anchor":"10153571"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fuavtjkb3klv6df\/","ref":"T371351#10153573"},[],{"anchor":"10153573"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_275\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fuavtjkb3klv6df\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_277\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fuavtjkb3klv6df","anchor":"10153573"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wk67himigfcf6ny\/","ref":"T371351#10153579"},[],{"anchor":"10153579"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_284\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wk67himigfcf6ny\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_286\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wk67himigfcf6ny","anchor":"10153579"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xialu76vfqxiiyy\/","ref":"T371351#10153582"},[],{"anchor":"10153582"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_293\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xialu76vfqxiiyy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_295\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xialu76vfqxiiyy","anchor":"10153582"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yry4fdatsph5j4x\/","ref":"T371351#10153584"},[],{"anchor":"10153584"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_302\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yry4fdatsph5j4x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_304\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-yry4fdatsph5j4x\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_306\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-yry4fdatsph5j4x","anchor":"10153584"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-l34yj2a5isg56sa\/","ref":"T371351#10153616"},[],{"anchor":"10153616"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_314\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_315\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l34yj2a5isg56sa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_316\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-l34yj2a5isg56sa","anchor":"10153616"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pm6w3idracsanco\/","ref":"T371351#10153650"},[],{"anchor":"10153650"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_325\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pm6w3idracsanco\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-pm6w3idracsanco","anchor":"10153650"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tde3rlclnp66xva\/","ref":"T371351#10153667"},[],{"anchor":"10153667"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_338\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tde3rlclnp66xva\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_340\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-tde3rlclnp66xva\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_342\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tde3rlclnp66xva","anchor":"10153667"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rw2ybqn2mm2jirw\/","ref":"T371351#10154105"},[],{"anchor":"10154105"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_350\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_351\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rw2ybqn2mm2jirw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_352\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_353\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rw2ybqn2mm2jirw","anchor":"10154105"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dm65kygkulu4aeo\/","ref":"T371351#10154152"},[],{"anchor":"10154152"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_361\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dm65kygkulu4aeo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_363\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_364\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-dm65kygkulu4aeo","anchor":"10154152"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nua4vqurbqtz2hf\/","ref":"T371351#10154179"},[],{"anchor":"10154179"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_374\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_375\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nua4vqurbqtz2hf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_376\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-nua4vqurbqtz2hf","anchor":"10154179"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kcbzccbpg7b6vmk\/","ref":"T371351#10156301"},[],{"anchor":"10156301"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_385\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kcbzccbpg7b6vmk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_387\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kcbzccbpg7b6vmk","anchor":"10156301"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-chxbsa2h524owtx","anchor":"10156341"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bxomttsu4mskq6y\/","ref":"T371351#10156417"},[],{"anchor":"10156417"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_397\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_398\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bxomttsu4mskq6y\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_399\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_400\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bxomttsu4mskq6y","anchor":"10156417"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-jfnibah6bszuvxm","anchor":"10156419"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wevkle75knat4t4\/","ref":"T371351#10156530"},[],{"anchor":"10156530"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_409\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wevkle75knat4t4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_411\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_412\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wevkle75knat4t4","anchor":"10156530"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cnvppia7cyna4t4\/","ref":"T371351#10156546"},[],{"anchor":"10156546"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_418\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cnvppia7cyna4t4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_420\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_421\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cnvppia7cyna4t4","anchor":"10156546"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-f4ww6oomj3q3gwb\/","ref":"T371351#10156630"},[],{"anchor":"10156630"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_427\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-f4ww6oomj3q3gwb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_429\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_430\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-f4ww6oomj3q3gwb","anchor":"10156630"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-nwj7c5ag7hwws5e","anchor":"10156822"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-wpocxfo2it7nul3","anchor":"10158996"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-q4hegfzzjaeixtm\/","ref":"T371351#10159982"},[],{"anchor":"10159982"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_443\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-q4hegfzzjaeixtm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_445\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_446\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-q4hegfzzjaeixtm","anchor":"10159982"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-apk56htzjbd2fo4\/","ref":"T371351#10159983"},[],{"anchor":"10159983"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_452\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-apk56htzjbd2fo4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_454\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-23fpomr5fuafev2","anchor":"10159983"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/237d637a\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/1767ee66\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cda2b7e5\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/aa49028c\/rsrc\/css\/phui\/phui-badge.css"]}