for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-sjaibs3bcs5nstmutlag\/?after=8032927&quoteTargetID=UQ0_1&quoteRef=T301949\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_230\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8032927\" id=\"8032927\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8032927\" data-sigil=\"has-tooltip\" data-meta=\"0_228\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 1:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 13:03:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_226\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_60\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/81VpqoEB6FQ6iqKiABhf\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_1\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-06-28T13:03:36Z] <taavi> grant the tools project access to the g3.cores16.ram64.disk20.10xiops flavor \u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_234\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8033000\" id=\"8033000\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_232\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003etaavi\u003c\/a\u003e mentioned this in \u003ca href=\"\/T311509\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003eT311509: Request increased quota for tools Cloud VPS project\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033000\" data-sigil=\"has-tooltip\" data-meta=\"0_231\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 1:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 13:18:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003etaavi\u003c\/a\u003e added a subtask: \u003ca href=\"\/T311509\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eT311509: Request increased quota for tools Cloud VPS project\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_237\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8033732\" id=\"8033732\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003etaavi\u003c\/a\u003e closed subtask \u003ca href=\"\/T311509\" class=\"phui-handle handle-status-closed\"\u003eT311509: Request increased quota for tools Cloud VPS project\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033732\" data-sigil=\"has-tooltip\" data-meta=\"0_235\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 3:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 15:51:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_246\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8033799\" id=\"8033799\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033799\" data-sigil=\"has-tooltip\" data-meta=\"0_244\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 4:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 16:07:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_242\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 809213 had a related patch set uploaded (by Majavah; author: Majavah):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] P:wmcs: toolsdb: support mariadb 10.4 on bullseye\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/809213\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/809213\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_249\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8033800\" id=\"8033800\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033800\" data-sigil=\"has-tooltip\" data-meta=\"0_247\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 4:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 16:07:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_258\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8033827\" id=\"8033827\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033827\" data-sigil=\"has-tooltip\" data-meta=\"0_256\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 4:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 16:12:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_254\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 809213 \u003cstrong\u003emerged\u003c\/strong\u003e by David Caro:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] P:wmcs: toolsdb: support mariadb 10.4 on bullseye\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/809213\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/809213\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_261\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8033866\" id=\"8033866\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033866\" data-sigil=\"has-tooltip\" data-meta=\"0_259\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 4:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 16:30:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_270\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8033872\" id=\"8033872\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033872\" data-sigil=\"has-tooltip\" data-meta=\"0_268\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 4:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 16:30:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_266\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/Jpwmq4EBa_6PSCT9x_Tl\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"4_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-06-28T16:30:53Z] <taavi> stopped mariadb on clouddb1002, starting rsync clouddb1002->tools-db-1 on a root screen session on tools-db-1 \u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_273\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8033890\" id=\"8033890\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_272\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003etaavi\u003c\/a\u003e claimed this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8033890\" data-sigil=\"has-tooltip\" data-meta=\"0_271\"\u003e\u003cspan class=\"screen-only\"\u003eJun 28 2022, 4:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-28 16:37:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_284\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_282\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8035850\" id=\"8035850\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003etaavi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8035850\" data-sigil=\"has-tooltip\" data-meta=\"0_280\"\u003e\u003cspan class=\"screen-only\"\u003eJun 29 2022, 6:38 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-06-29 06:38:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_278\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_279\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_78\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003etools-db-1.tools.eqiad1.wikimedia.cloud now is now replicating. The only problem so far is that users and grants didn't seem to get copied over for some reason.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_287\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8143314\" id=\"8143314\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_286\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003ebd808\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/539\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/539\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003eIn Progress\u003c\/a\u003e on the \u003ca href=\"\/tag\/toolforge\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eToolforge\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8143314\" data-sigil=\"has-tooltip\" data-meta=\"0_285\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2022, 5:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-08-10 17:03:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_296\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v2iza5xhmlrh6nzci6b6\/PHID-FILE-jtmoofc7k3odr7cyyhyu\/profile-martian.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Andrew\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8153548\" id=\"8153548\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003eAndrew\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8153548\" data-sigil=\"has-tooltip\" data-meta=\"0_294\"\u003e\u003cspan class=\"screen-only\"\u003eAug 15 2022, 1:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-08-15 13:01:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_292\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAmong the various ways to get this properly replicating on Ceph, one option is throttling the rate of user access until we get below our latency needs:\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/www.percona.com\/blog\/2020\/10\/02\/rate-limit-throttle-for-mysql-with-proxysql\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/www.percona.com\/blog\/2020\/10\/02\/rate-limit-throttle-for-mysql-with-proxysql\/\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_299\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8185781\" id=\"8185781\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003efnegri\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8185781\" data-sigil=\"has-tooltip\" data-meta=\"0_297\"\u003e\u003cspan class=\"screen-only\"\u003eAug 25 2022, 3:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-08-25 15:52:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_308\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v2iza5xhmlrh6nzci6b6\/PHID-FILE-jtmoofc7k3odr7cyyhyu\/profile-martian.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Andrew\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8262117\" id=\"8262117\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eAndrew\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8262117\" data-sigil=\"has-tooltip\" data-meta=\"0_306\"\u003e\u003cspan class=\"screen-only\"\u003eSep 26 2022, 6:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-26 18:54:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_304\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@taavi\u003c\/span\u003e\u003c\/a\u003e can you catch me up on the state of this task? Are still still mostly blocked by high ceph latency or did your comment at the end of June imply that you're past that?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_317\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v2iza5xhmlrh6nzci6b6\/PHID-FILE-jtmoofc7k3odr7cyyhyu\/profile-martian.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Andrew\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8262123\" id=\"8262123\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003eAndrew\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8262123\" data-sigil=\"has-tooltip\" data-meta=\"0_315\"\u003e\u003cspan class=\"screen-only\"\u003eSep 26 2022, 6:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-26 18:57:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_313\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ebtw, I just checked the spreadsheets and our last local-storage hypervisors are scheduled for refresh in 2023.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_328\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_326\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_327\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8266105\" id=\"8266105\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003etaavi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8266105\" data-sigil=\"has-tooltip\" data-meta=\"0_324\"\u003e\u003cspan class=\"screen-only\"\u003eSep 27 2022, 7:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-27 19:12:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_322\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T301949#8262117\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949#8262117\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Andrew\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Andrew\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@taavi\u003c\/span\u003e\u003c\/a\u003e can you catch me up on the state of this task? Are still still mostly blocked by high ceph latency or did your comment at the end of June imply that you're past that?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI think that comment was before I discovered Ceph latency was an issue. I don't have any actual evidence of this, but somehow things felt _slower_ after you tuned the flavor settings.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_339\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_337\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_338\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8266106\" id=\"8266106\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003etaavi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8266106\" data-sigil=\"has-tooltip\" data-meta=\"0_335\"\u003e\u003cspan class=\"screen-only\"\u003eSep 27 2022, 7:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-27 19:13:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_333\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnyhow, I suspect the bullseye replica would need to be re-cloned from the replica since it's been so long without being able to catch up on replication.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_342\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/37i6uiqajx6dfu7b376y\/PHID-FILE-rejjkqhpflbdwlrpit6o\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Albertoleoncio\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8266203\" id=\"8266203\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Albertoleoncio\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003eAlbertoleoncio\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8266203\" data-sigil=\"has-tooltip\" data-meta=\"0_340\"\u003e\u003cspan class=\"screen-only\"\u003eSep 27 2022, 7:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-27 19:45:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_345\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/bov64zymbxsy2emzxaut\/PHID-FILE-n37ejbchno4lp5hlru3f\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dcaro\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8287551\" id=\"8287551\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_344\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dcaro\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_95\"\u003edcaro\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8287551\" data-sigil=\"has-tooltip\" data-meta=\"0_343\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2022, 4:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-05 16:10:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_354\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8314788\" id=\"8314788\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_353\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8314788\" data-sigil=\"has-tooltip\" data-meta=\"0_352\"\u003e\u003cspan class=\"screen-only\"\u003eOct 13 2022, 4:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-13 16:00:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_350\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_351\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI'm going to spend some time on this task this month, as it's one of the WMCS priorities for this quarter. As a start, I was thinking of having another shot at setting up a new replica using Ceph storage. I have some questions about the previous failed attempt (tools-db-1):\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHow long did the rsync from clouddb1002 to tools-db-1 take? Was clouddb1002 stopped for the whole time, or did the clone use a \u003ca href=\"https:\/\/www.electricmonk.nl\/log\/2016\/11\/06\/very-fast-mysql-slave-setup-with-zero-downtime-using-rsync\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003edouble rsync approach\u003c\/a\u003e?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWas the replication in tools-db-1 started using GTID or binary log (see \u003ca href=\"https:\/\/mariadb.com\/kb\/en\/setting-up-a-replica-with-mariabackup\/#start-replication-on-the-new-replica\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003edocs\u003c\/a\u003e)?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWas tools-db-1 ever able to catch up with the primary? Did the replication lag get to zero then increased, or did it never get to zero?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf I wanted to see the metrics for tools-db-1 in \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/000000273\/tools-mariadb\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003egrafana-labs\u003c\/a\u003e, would I have to configure something?\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eAn additional note: according to \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Help:Toolforge\/Database#ToolsDB_Backups_and_Replication\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethis page\u003c\/a\u003e some databases in clouddb1001 are \u003cem\u003enot\u003c\/em\u003e replicated to 1002. That means for a real migration we will have to clone from the primary (1001), or find a way to migrate those dbs separately. We might even consider setting up a separate VM just for those dbs?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_365\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_363\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_364\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8314988\" id=\"8314988\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003etaavi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8314988\" data-sigil=\"has-tooltip\" data-meta=\"0_361\"\u003e\u003cspan class=\"screen-only\"\u003eOct 13 2022, 4:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-13 16:33:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_359\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_360\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T301949#8314788\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949#8314788\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fnegri\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fnegri\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI'm going to spend some time on this task this month, as it's one of the WMCS priorities for this quarter.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eGreat!\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eAs a start, I was thinking of having another shot at setting up a new replica using Ceph storage. I have some questions about the previous failed attempt (tools-db-1):\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHow long did the rsync from clouddb1002 to tools-db-1 take? Was clouddb1002 stopped for the whole time, or did the clone use a \u003ca href=\"https:\/\/www.electricmonk.nl\/log\/2016\/11\/06\/very-fast-mysql-slave-setup-with-zero-downtime-using-rsync\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003edouble rsync approach\u003c\/a\u003e?\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI believe I started it in my afternoon and it was finished when I checked on it the next morning. So something like 12-24 hours. I can try to dig up the IRC logs if you want a more detailed answer. MariaDB was stopped during the transfer.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003col class=\"remarkup-list\" start=\"2\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWas the replication in tools-db-1 started using GTID or binary log (see \u003ca href=\"https:\/\/mariadb.com\/kb\/en\/setting-up-a-replica-with-mariabackup\/#start-replication-on-the-new-replica\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003edocs\u003c\/a\u003e)?\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eGTID.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003col class=\"remarkup-list\" start=\"3\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWas tools-db-1 ever able to catch up with the primary? Did the replication lag get to zero then increased, or did it never get to zero?\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eNo, it was constantly increasing.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003col class=\"remarkup-list\" start=\"4\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf I wanted to see the metrics for tools-db-1 in \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/000000273\/tools-mariadb\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003egrafana-labs\u003c\/a\u003e, would I have to configure something?\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eYes, you'd need to configure either the tools or the [[ Nova Resource:Metricsinfra | metricsinfra ]] prometheus servers (or both) to scrape it, and then you'd need to update the dashboard as IIRC it currently filters on the clouddb-services project.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eAn additional note: according to \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Help:Toolforge\/Database#ToolsDB_Backups_and_Replication\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethis page\u003c\/a\u003e some databases in clouddb1001 are \u003cem\u003enot\u003c\/em\u003e replicated to 1002. That means for a real migration we will have to clone from the primary (1001), or find a way to migrate those dbs separately. We might even consider setting up a separate VM just for those dbs?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eYeah. And I'm not sure how much I trust that 1002 is an accurate replica as we've had a few issues with it in the last few years, so cloning from the primary might be the only option.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_374\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8317377\" id=\"8317377\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_373\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8317377\" data-sigil=\"has-tooltip\" data-meta=\"0_372\"\u003e\u003cspan class=\"screen-only\"\u003eOct 14 2022, 3:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-14 15:55:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_370\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_101\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks \u003ca href=\"\/p\/taavi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@taavi\u003c\/span\u003e\u003c\/a\u003e -- here's what I'm planning to do. I would like to replicate your experiment but with a couple of small differences:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI'll leave clouddb1002 running during the rsync, hopefully the disk in 1002 is fast enough to rsync while replicating from the primary. Then I'll stop 1002 briefly, and run rsync again, which should complete quickly, with a much smaller downtime. This is also a test to see if the same strategy could be used for cloning the primary database later on.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI'll create a new test instance \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e to start completely from scratch, and I'll make sure I can see the MariaDB metrics for that instance in \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/000000273\/tools-mariadb\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003egrafana-labs\u003c\/a\u003e before starting the replication.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI'm expecting to encounter the exact same problem \u003ca href=\"\/p\/taavi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@taavi\u003c\/span\u003e\u003c\/a\u003e described where the replication lag will be constantly increasing, but I hope the metrics might give us some interesting data point to consider, and help us clarify if Ceph is really too slow for the replication to work. If this experiment won't give us any useful insight, I have discussed a few other options with \u003ca href=\"\/p\/Andrew\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Andrew\u003c\/span\u003e\u003c\/a\u003e but I will write about them later.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003esomething like 12-24 hours. I can try to dig up the IRC logs if you want a more detailed answer.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eNo need to look in the logs, the ballpark 12-24h is good enough! It would be useful if you remember how you connected from tools-db-1 to clouddb1002 (SSH? Which key?) and any rsync option you used. Another approach could be using the Puppet class \u003ctt class=\"remarkup-monospaced\"\u003ersync::quickdatacopy\u003c\/tt\u003e as done in \u003ca href=\"\/T266587\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT266587\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eYes, you'd need to configure either the tools or the \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Nova_Resource:Metricsinfra\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003emetricsinfra\u003c\/a\u003e prometheus servers (or both) to scrape it, and then you'd need to update the dashboard as IIRC it currently filters on the clouddb-services project.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI checked \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/prometheus\/cloud\/prometheus.yml\u003c\/tt\u003e in \u003ctt class=\"remarkup-monospaced\"\u003emetricsinfra-prometheus-2\u003c\/tt\u003e and it seems to be scraping \u003ctt class=\"remarkup-monospaced\"\u003etools-db-\\d+\u003c\/tt\u003e already, in addition to \u003ctt class=\"remarkup-monospaced\"\u003eclouddb100[12]\u003c\/tt\u003e. And indeed I can find \u003ctt class=\"remarkup-monospaced\"\u003emysql_up{instance="tools-db-1"}\u003c\/tt\u003e in grafana-cloud, I guess the other metrics are missing only because the database is currently stopped.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_383\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8326055\" id=\"8326055\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_382\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8326055\" data-sigil=\"has-tooltip\" data-meta=\"0_381\"\u003e\u003cspan class=\"screen-only\"\u003eOct 18 2022, 4:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-18 16:00:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_379\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_380\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI did some disk performance tests on 3 new VMs in the testlabs project, \u003ctt class=\"remarkup-monospaced\"\u003elocal-storage-test-stretch\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003elocal-storage-test-bullseye\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003eceph-storage-test\u003c\/tt\u003e. I have used \u003ctt class=\"remarkup-monospaced\"\u003efio\u003c\/tt\u003e following \u003ca href=\"https:\/\/cloud.google.com\/compute\/docs\/disks\/benchmarking-pd-performance\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethese examples I found in the Google Cloud docs\u003c\/a\u003e, and these are the results.\u003c\/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eSequential writes\u003c\/strong\u003e test:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003elocal-storage-test-stretch: 1241.1MB\/s\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003elocal-storage-test-bullseye: 1334MB\/s\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eceph-storage-test: 206MB\/s\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e\u003cstrong\u003eRandom writes\u003c\/strong\u003e test:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003elocal-storate-test-stretch: 74409KB\/s\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003elocal-storage-test-bullseye: 181MB\/s\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eceph-storage-test: 2036kB\/s\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAs expected Ceph is much slower, but it's not clear if it's slow enough to explain the increasing replication lag that was seen in tools-db-1. On clouddb1002, \u003ctt class=\"remarkup-monospaced\"\u003eiostat -xdmy 5\u003c\/tt\u003e is showing a current write load of about 4MB\/s, and \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/000000273\/tools-mariadb?orgId=1&from=now-24h&to=now&var-server=clouddb1002\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eGrafana\u003c\/a\u003e is showing a peak of about 60MB\/s in the past 24 hours.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_392\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8328457\" id=\"8328457\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_391\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8328457\" data-sigil=\"has-tooltip\" data-meta=\"0_390\"\u003e\u003cspan class=\"screen-only\"\u003eOct 19 2022, 10:35 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-19 10:35:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_388\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_389\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI also checked if there was any performance difference between \u003ctt class=\"remarkup-monospaced\"\u003eceph-storage-test\u003c\/tt\u003e (using flavor g3.cores1.ram2.disk20) and \u003ctt class=\"remarkup-monospaced\"\u003etools-db-1\u003c\/tt\u003e (using     g3.cores16.ram64.disk20.maxiops). From what I could see with \u003ctt class=\"remarkup-monospaced\"\u003efio\u003c\/tt\u003e, the \u003ctt class=\"remarkup-monospaced\"\u003emaxiops\u003c\/tt\u003e flavor does not lead to any visible difference in bandwidth or IOPS:\u003c\/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eSequential writes\u003c\/strong\u003e:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eceph-storage-test: \u003ctt class=\"remarkup-monospaced\"\u003ewrite: IOPS=186, BW=195MiB\/s (204MB\/s)(11.7GiB\/61696msec)\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003etools-db-1: \u003ctt class=\"remarkup-monospaced\"\u003ewrite: IOPS=182, BW=191MiB\/s (200MB\/s)(11.7GiB\/62681msec)\u003c\/tt\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e\u003cstrong\u003eRandom writes\u003c\/strong\u003e:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eceph-storaget-test: \u003ctt class=\"remarkup-monospaced\"\u003ewrite: IOPS=496, BW=1989KiB\/s (2037kB\/s)(117MiB\/60127msec)\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003etools-db-1: \u003ctt class=\"remarkup-monospaced\"\u003ewrite: IOPS=493, BW=1979KiB\/s (2027kB\/s)(116MiB\/60127msec)\u003c\/tt\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/dcaro\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@dcaro\u003c\/span\u003e\u003c\/a\u003e reminded me we have more Ceph performance tests available at \u003ca href=\"https:\/\/cloud-ceph-performance-tests.toolforge.org\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/cloud-ceph-performance-tests.toolforge.org\/\u003c\/a\u003e and they seem to confirm values around 200MB\/s on sequential writes, and around 2MB\/s on random writes.\u003c\/p\u003e\n\n\u003cp\u003eI don't think it's easy to say if these values are good enough for our database load, but my hunch is that unless MariaDB is writing in a purely sequential fashion (unlikely), Ceph won't be fast enough to cope with our current load peaks that according to Grafana are of about 60MB\/s and 2k IOPS on clouddb1002 (and even higher on clouddb1001).\u003c\/p\u003e\n\n\u003cp\u003eI will anyway proceed with setting up a new replica \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e to collect more data under a real database load.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_395\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8338540\" id=\"8338540\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_394\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003efnegri\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8338540\" data-sigil=\"has-tooltip\" data-meta=\"0_393\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 1:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 13:05:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_404\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8345255\" id=\"8345255\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_403\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_107\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8345255\" data-sigil=\"has-tooltip\" data-meta=\"0_402\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2022, 11:20 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-26 11:20:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_400\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have started the \u003ctt class=\"remarkup-monospaced\"\u003ersync\u003c\/tt\u003e from \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e to \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e using the Puppet class \u003ctt class=\"remarkup-monospaced\"\u003ersync::quickdatacopy\u003c\/tt\u003e which creates \u003ctt class=\"remarkup-monospaced\"\u003e\/usr\/local\/sbin\/sync-srv-labsdb-backup1\u003c\/tt\u003e. I have launched that script from a \u003ctt class=\"remarkup-monospaced\"\u003escreen\u003c\/tt\u003e session in \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e. I also had to add a Security Group rule in Horizon to allow connections on the Rsync port from \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e.\u003c\/p\u003e\n\n\u003cp\u003eThe Rsync is expected to take 12-24 hours. MariaDB in \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e is still running and seems to be coping well with the extra disk load.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_407\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8350470\" id=\"8350470\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_406\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003efnegri\u003c\/a\u003e added a subtask: \u003ca href=\"\/T321828\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003eT321828: [ceph] Tune performance for DB VMs\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8350470\" data-sigil=\"has-tooltip\" data-meta=\"0_405\"\u003e\u003cspan class=\"screen-only\"\u003eOct 27 2022, 4:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-27 16:21:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_416\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8350981\" id=\"8350981\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_415\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003efnegri\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003eaborrero\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8350981\" data-sigil=\"has-tooltip\" data-meta=\"0_414\"\u003e\u003cspan class=\"screen-only\"\u003eOct 27 2022, 6:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-27 18:58:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_412\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_413\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_113\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eRsync completed in about 9 hours, transferring 2.9TB at an approximate speed of 92MB\/s, which is lower than what we saw in performance tests for sequential writes. \u003ca href=\"\/p\/dcaro\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@dcaro\u003c\/span\u003e\u003c\/a\u003e created the subtask \u003ca href=\"\/T321828\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT321828\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e to explore potential ways to improve the speed of Ceph volumes.\u003c\/p\u003e\n\n\u003cp\u003eI'm now trying to do incremental rsyncs with the aim of getting the data in tools-db-2 close enough to clouddb1002, so that when I start MariaDB in tools-db-2 it won't have too many hours to catch up. To do effective incremental rsyncs I had to tune the rsync command and add \u003ctt class=\"remarkup-monospaced\"\u003e--delete\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003e-W\u003c\/tt\u003e to disable rsync's "delta-transfer algorithm" that was slowing things down. One obstacle to fast incremental rsyncs is a set of very big files that (when modified) need to be rsynced again. The biggest one is \u003ctt class=\"remarkup-monospaced\"\u003edata\/s51230__linkwatcher\/linkwatcher_linklog.ibd\u003c\/tt\u003e which at 1.1TB of size is about a third of the total database size. :D This was already spotted by \u003ca href=\"\/p\/aborrero\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aborrero\u003c\/span\u003e\u003c\/a\u003e in \u003ca href=\"\/T301967\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301967\u003c\/span\u003e\u003c\/a\u003e a few months ago, the size has increased by 70GB since then (about 9GB per month) and this database seems a good candidate for being split to a dedicated server.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_429\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_425\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_427\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8352177\" id=\"8352177\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8352177\" data-sigil=\"has-tooltip\" data-meta=\"0_423\"\u003e\u003cspan class=\"screen-only\"\u003eOct 28 2022, 10:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-28 10:33:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_421\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_422\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T301949#8350981\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949#8350981\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fnegri\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fnegri\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eOne obstacle to fast incremental rsyncs is a set of very big files that (when modified) need to be rsynced again. The biggest one is \u003ctt class=\"remarkup-monospaced\"\u003edata\/s51230__linkwatcher\/linkwatcher_linklog.ibd\u003c\/tt\u003e which at 1.1TB of size is about a third of the total database size. :D This was already spotted by \u003ca href=\"\/p\/aborrero\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aborrero\u003c\/span\u003e\u003c\/a\u003e in \u003ca href=\"\/T301967\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301967\u003c\/span\u003e\u003c\/a\u003e a few months ago, the size has increased by 70GB since then (about 9GB per month) and this database seems a good candidate for being split to a dedicated server.\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThat table size is seriously big. I would suggest to talk to the tool owner to see if it can be reduced (or optimized at least, in case there're lots of blank spaces on that table). But that's going to cause (more) issues in the future if the size isn't reduced.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_440\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_438\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_439\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8353450\" id=\"8353450\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8353450\" data-sigil=\"has-tooltip\" data-meta=\"0_436\"\u003e\u003cspan class=\"screen-only\"\u003eOct 28 2022, 5:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-28 17:16:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_434\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T301949#8352177\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949#8352177\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T301949#8350981\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949#8350981\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fnegri\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fnegri\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eOne obstacle to fast incremental rsyncs is a set of very big files that (when modified) need to be rsynced again. The biggest one is \u003ctt class=\"remarkup-monospaced\"\u003edata\/s51230__linkwatcher\/linkwatcher_linklog.ibd\u003c\/tt\u003e which at 1.1TB of size is about a third of the total database size. :D This was already spotted by \u003ca href=\"\/p\/aborrero\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aborrero\u003c\/span\u003e\u003c\/a\u003e in \u003ca href=\"\/T301967\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301967\u003c\/span\u003e\u003c\/a\u003e a few months ago, the size has increased by 70GB since then (about 9GB per month) and this database seems a good candidate for being split to a dedicated server.\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThat table size is seriously big. I would suggest to talk to the tool owner to see if it can be reduced (or optimized at least, in case there're lots of blank spaces on that table). But that's going to cause (more) issues in the future if the size isn't reduced.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/T224154\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT224154: Reduce size of linkwatcher db if at all possible\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_443\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8380161\" id=\"8380161\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003etaavi\u003c\/a\u003e reassigned this task from \u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\"\u003etaavi\u003c\/a\u003e to \u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\"\u003efnegri\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8380161\" data-sigil=\"has-tooltip\" data-meta=\"0_441\"\u003e\u003cspan class=\"screen-only\"\u003eNov 8 2022, 4:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-08 16:25:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_452\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8383081\" id=\"8383081\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_119\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8383081\" data-sigil=\"has-tooltip\" data-meta=\"0_450\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2022, 1:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-09 13:59:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_448\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_120\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI found some interesting new data: Ceph is not the only bottleneck for rsync from \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e to \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e. Rsyncing to a \u003ctt class=\"remarkup-monospaced\"\u003etmpfs\u003c\/tt\u003e volume gets is comparably fast (around 100Mbps). There is some interesting caching going on though, because if I rsync the same file twice in a row, the second time is 2x to 3x faster, both writing to Ceph or to the tmpfs volume.\u003c\/p\u003e\n\n\u003cp\u003eI reproduced this with a few different files I picked randomly among the ones between 5GB and 10GB in size (5GB and 10GB were also arbitrary values):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003emount -o size=16G -t tmpfs none \/mnt\/tmpfs\n\n# tmpfs, no cache\nroot@tools-db-2:~# \/usr\/bin\/rsync --info=stats2,progress2 -aW rsync:\/\/clouddb1002.clouddb-services.eqiad.wmflabs\/srv-labsdb-backup1\/data\/s51512__data\/dewiki_val.ibd \/mnt\/tmpfs\/\n  6,517,948,416 100%   97.49MB\/s    0:01:03 (xfr#1, to-chk=0\/1)\n\n# tmpfs, cached\nroot@tools-db-2:~# rm \/mnt\/tmpfs\/dewiki_val.ibd\nroot@tools-db-2:~# \/usr\/bin\/rsync --info=stats2,progress2 -aW rsync:\/\/clouddb1002.clouddb-services.eqiad.wmflabs\/srv-labsdb-backup1\/data\/s51512__data\/dewiki_val.ibd \/mnt\/tmpfs\/\n  6,517,948,416 100%  294.72MB\/s    0:00:21 (xfr#1, to-chk=0\/1)\n\n# Ceph, cached\nroot@tools-db-2:~# \/usr\/bin\/rsync --info=stats2,progress2 -aW rsync:\/\/clouddb1002.clouddb-services.eqiad.wmflabs\/srv-labsdb-backup1\/data\/s51512__data\/dewiki_val.ibd \/srv\/labsdb\/\n  6,517,948,416 100%  237.09MB\/s    0:00:26 (xfr#1, to-chk=0\/1)\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_455\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8383210\" id=\"8383210\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_454\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_121\"\u003efnegri\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/2774\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003eNeeds discussion\u003c\/a\u003e to \u003ca href=\"\/project\/board\/2774\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_123\"\u003eDoing\u003c\/a\u003e on the \u003ca href=\"\/project\/view\/2774\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003ecloud-services-team (Kanban)\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8383210\" data-sigil=\"has-tooltip\" data-meta=\"0_453\"\u003e\u003cspan class=\"screen-only\"\u003eNov 9 2022, 2:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-09 14:41:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_464\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8389718\" id=\"8389718\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_463\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8389718\" data-sigil=\"has-tooltip\" data-meta=\"0_462\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2022, 4:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-11 16:12:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_460\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_461\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_126\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/rwF3Z4QB6FQ6iqKiK5AS\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"4_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-11-11T16:12:53Z] <dhinus> stopping mariadb for a few minutes on replica server clouddb1002 (\u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_473\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8389719\" id=\"8389719\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_472\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_127\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8389719\" data-sigil=\"has-tooltip\" data-meta=\"0_471\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2022, 4:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-11 16:14:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_469\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_470\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_128\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/utF4Z4QB8Fs0LHO5XPOQ\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"4_4\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-11-11T16:14:12Z] <dhinus> rsyncing a single database from clouddb1002 to tools-db-2 (as a test for \u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_482\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8389909\" id=\"8389909\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_481\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_129\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8389909\" data-sigil=\"has-tooltip\" data-meta=\"0_480\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2022, 5:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-11 17:37:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_478\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_479\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_130\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/4EbEZ4QBa_6PSCT9ahma\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"4_5\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-11-11T17:37:15Z] <dhinus> mariadb restarted on clouddb1002 and again in sync with primary (took a bit to catch up) \u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_492\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8390250\" id=\"8390250\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_491\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_131\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8390250\" data-sigil=\"has-tooltip\" data-meta=\"0_490\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2022, 10:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-11 22:21:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_488\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_489\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_132\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAs a test, I tried setting up \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e to replicate only a single database from \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e. I picked \u003ctt class=\"remarkup-monospaced\"\u003es51138__heritage_p\u003c\/tt\u003e because in \u003ca href=\"\/T307855\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT307855\u003c\/span\u003e\u003c\/a\u003e it was noted that it's making a lot of write queries.\u003c\/p\u003e\n\n\u003cp\u003eInitially I tried using rsync to copy only the files related to that database, but that didn't work (probably because some information is also stored in the \u003ctt class=\"remarkup-monospaced\"\u003eibdata1\u003c\/tt\u003e file and maybe others), so I used \u003ca href=\"https:\/\/mariadb.com\/kb\/en\/setting-up-a-replica-with-mariabackup\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethis strategy\u003c\/a\u003e instead, adding \u003ctt class=\"remarkup-monospaced\"\u003e--databases=s51138__heritage_p\u003c\/tt\u003e to the \u003ctt class=\"remarkup-monospaced\"\u003emariabackup\u003c\/tt\u003e commands:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clouddb1002:~# mariabackup --backup --target-dir=\/srv\/labsdb\/backup_test_T301949 --databases=s51138__heritage_p --user root\nroot@clouddb1002:~# mariabackup --prepare --target-dir=\/srv\/labsdb\/backup_test_T301949 --databases=s51138__heritage_p\n\nroot@tools-db-2:~# \/usr\/bin\/rsync -i -aW -n rsync:\/\/clouddb1002.clouddb-services.eqiad.wmflabs\/srv-labsdb-backup1\/backup_test_T301949\/ \/srv\/labsdb\/data\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI then configured \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e to replicate only that database using \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e as the source\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSET GLOBAL replicate_do_db='s51138__heritage_p';\nSET GLOBAL gtid_slave_pos = '{{value_from_the_file_xtrabackup_binlog_info}}'\nCHANGE MASTER TO MASTER_HOST='clouddb1002.clouddb-services.eqiad1.wikimedia.cloud', MASTER_USER='repl', MASTER_PASSWORD='*******' ', MASTER_USE_GTID=slave_pos;\nSTART SLAVE;\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eEven with only one database to replicate, it looks like tools-db-2 is struggling to catch up:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ermtvjirnbsazzrj6qgw\/PHID-FILE-r3dejxze7aumfqa3ln46\/Screenshot_2022-11-11_at_23.09.20.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_34\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/p6cyihndop33n6zv3bxm\/PHID-FILE-mez4anpwnrbs2k7nxdtj\/preview-Screenshot_2022-11-11_at_23.09.20.png\" width=\"220\" height=\"209.60863697706\" alt=\"Screenshot 2022-11-11 at 23.09.20.png (1\u00d71 px, 652 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/lzrt7677v2nsx5jkerll\/PHID-FILE-hszernfidfsklhzxkrem\/Screenshot_2022-11-11_at_23.09.40.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_35\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/5a77v7zl5sapgm7z6hns\/PHID-FILE-uycv4tgost7244jmz5pl\/preview-Screenshot_2022-11-11_at_23.09.40.png\" width=\"220\" height=\"140.50420168067\" alt=\"Screenshot 2022-11-11 at 23.09.40.png (1\u00d71 px, 521 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eI will leave it running and check again tomorrow.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_501\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8390668\" id=\"8390668\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_500\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_133\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8390668\" data-sigil=\"has-tooltip\" data-meta=\"0_499\"\u003e\u003cspan class=\"screen-only\"\u003eNov 12 2022, 9:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-12 21:30:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_497\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_498\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNot looking great 24h later:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/hdo63p7y7v7vfbrdhlam\/PHID-FILE-z2jedi7oox7w3n66hoje\/Screenshot_2022-11-12_at_22.30.00.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_36\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/e2wddjgrnkhzhes5nsz5\/PHID-FILE-rekx4w2xcy5y6pvlaff5\/preview-Screenshot_2022-11-12_at_22.30.00.png\" width=\"220\" height=\"179.49367088608\" alt=\"Screenshot 2022-11-12 at 22.30.00.png (1\u00d71 px, 247 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_514\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_510\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_511\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_512\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_513\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8391199\" id=\"8391199\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_509\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_135\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8391199\" data-sigil=\"has-tooltip\" data-meta=\"0_508\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2022, 6:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-14 06:40:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_506\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_507\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_136\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCan I get tools-db-2 full hostname so I can ssh into it?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_523\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8391522\" id=\"8391522\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_522\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_137\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8391522\" data-sigil=\"has-tooltip\" data-meta=\"0_521\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2022, 9:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-14 09:48:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_519\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_520\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_138\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2.tools.eqiad1.wikimedia.cloud\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_532\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8391524\" id=\"8391524\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_531\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_139\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8391524\" data-sigil=\"has-tooltip\" data-meta=\"0_530\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2022, 9:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-14 09:49:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_528\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_529\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_140\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI see a pattern there: \ud83d\ude04\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/bgmn5kfpdx362vkhipap\/PHID-FILE-cfmqff4ajcmnmut3govv\/Screenshot_2022-11-14_at_10.49.14.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_38\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/qjpb6kz7xdctii3hgfss\/PHID-FILE-45ik5nxap6zew4anyprs\/preview-Screenshot_2022-11-14_at_10.49.14.png\" width=\"220\" height=\"109.35217903416\" alt=\"Screenshot 2022-11-14 at 10.49.14.png (844\u00d71 px, 282 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_541\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8394082\" id=\"8394082\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_540\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_141\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8394082\" data-sigil=\"has-tooltip\" data-meta=\"0_539\"\u003e\u003cspan class=\"screen-only\"\u003eNov 14 2022, 8:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-14 20:37:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_537\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_538\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_142\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eA few more graphs below (timeframe is last 7 days for all).\u003c\/p\u003e\n\n\u003cp\u003eIOPS on clouddb1002:\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/cptxioonrxhlda2jzo7a\/PHID-FILE-4pbdwzvuc2xr4hvudql4\/Screenshot_2022-11-14_at_21.31.32.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_39\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/gog6rzdb6y5kqcvsx3xb\/PHID-FILE-vs7l6ra5p4ie6dly4ftj\/preview-Screenshot_2022-11-14_at_21.31.32.png\" width=\"220\" height=\"101.6\" alt=\"Screenshot 2022-11-14 at 21.31.32.png (762\u00d71 px, 577 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eIOPS on tools-db-2 (seems throttled at 500 IOPS):\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/33xouc5mi7zwpux2ffnq\/PHID-FILE-ahjk463wc3pnhmcs7wix\/Screenshot_2022-11-14_at_21.32.16.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_40\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/77umh2heop74eor3ns7v\/PHID-FILE-4jkr4nkrb4rv4itwaxcg\/preview-Screenshot_2022-11-14_at_21.32.16.png\" width=\"220\" height=\"109.86568986569\" alt=\"Screenshot 2022-11-14 at 21.32.16.png (818\u00d71 px, 150 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eWrite latency on clouddb1002 (yellow) vs tools-db-2 (green):\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/s5ol5lnccl7l7l7j4upg\/PHID-FILE-ag7hkupi4htfx5lj24ky\/Screenshot_2022-11-14_at_21.32.48.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_41\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/6duwbdf7bbl6mdnpr2ts\/PHID-FILE-wol2mbodcbopsti7d35n\/preview-Screenshot_2022-11-14_at_21.32.48.png\" width=\"220\" height=\"104.59378134403\" alt=\"Screenshot 2022-11-14 at 21.32.48.png (948\u00d71 px, 586 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eI will experiment tweaking the OpenStack \u003ca href=\"https:\/\/docs.openstack.org\/nova\/rocky\/user\/flavors.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eflavors\u003c\/a\u003e to see if I can remove the IOPS throttling. A previous attempt \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T301949#8266105\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ewas unsuccessful\u003c\/a\u003e so I wonder if the throttling is coming from the flavor, or somewhere else.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_554\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_550\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_551\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_552\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_553\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8395173\" id=\"8395173\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_549\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_143\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8395173\" data-sigil=\"has-tooltip\" data-meta=\"0_548\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 9:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 09:33:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_546\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_547\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_144\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSo it looks like that replicating only \u003ctt class=\"remarkup-monospaced\"\u003e               Replicate_Do_DB: s51138__heritage_p\u003c\/tt\u003e the host was able to catch up?\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 73\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eLooks like it is always around 60-80 seconds delayed, which I think is acceptable?\u003cbr \/\u003e\nHowever, we'd need to see how it goes if once recloned again and started to replicate everything again\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_567\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_563\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_565\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8395192\" id=\"8395192\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_562\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_145\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8395192\" data-sigil=\"has-tooltip\" data-meta=\"0_561\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 9:42 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 09:42:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_559\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_560\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_146\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe could also try to changing \u003ctt class=\"remarkup-monospaced\"\u003einnodb_flush_log_at_trx_commit\u003c\/tt\u003e from 1 to 2, which is not as data safe, but still safe enough and can probably get us some IOPS back.\u003cbr \/\u003e\nThis is not for me to decide, but for WMCS, so please make sure you read and understand the implications:\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/dev.mysql.com\/doc\/refman\/5.7\/en\/innodb-parameters.html#sysvar_innodb_flush_log_at_trx_commit\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/dev.mysql.com\/doc\/refman\/5.7\/en\/innodb-parameters.html#sysvar_innodb_flush_log_at_trx_commit\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eThe default setting of 1 is required for full ACID compliance. Logs are written and flushed to disk at each transaction commit.\n\nWith a setting of 0, logs are written and flushed to disk once per second. Transactions for which logs have not been flushed can be lost in a crash.\n\nWith a setting of 2, logs are written after each transaction commit and flushed to disk once per second. Transactions for which logs have not been flushed can be lost in a crash.\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_576\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8395372\" id=\"8395372\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_575\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_147\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8395372\" data-sigil=\"has-tooltip\" data-meta=\"0_574\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 10:42 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 10:42:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_572\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_148\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eLooks like it is always around 60-80 seconds delayed, which I think is acceptable?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIt was able to catch up, but whenever there's a bit of traffic on that database, the lag increases up to 20,000 seconds, following a very regular pattern:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/u4wx6flecr25hnbp6hb7\/PHID-FILE-73xmazrrh2optmevbafi\/Screenshot_2022-11-15_at_11.41.46.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_42\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/7oy2nhl3ahqfyrogolhx\/PHID-FILE-o3vhzxkt4bn57jvl5b7p\/preview-Screenshot_2022-11-15_at_11.41.46.png\" width=\"220\" height=\"106.83778234086\" alt=\"Screenshot 2022-11-15 at 11.41.46.png (946\u00d71 px, 185 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eI think it's mostly caused by the IOPS being throttled.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWe could also try to changing innodb_flush_log_at_trx_commit from 1 to 2, which is not as data safe, but still safe enough and can probably get us some IOPS back.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThanks, didn't know about this param, could be useful!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_585\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8395518\" id=\"8395518\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_584\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_149\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8395518\" data-sigil=\"has-tooltip\" data-meta=\"0_583\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 11:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 11:23:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_581\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_582\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe flavor of \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e is already set to \u003ctt class=\"remarkup-monospaced\"\u003eg3.cores16.ram64.disk20.10xiops\u003c\/tt\u003e which has \u003ctt class=\"remarkup-monospaced\"\u003equota:disk_read_iops_sec='50000', quota:disk_write_iops_sec='5000'\u003c\/tt\u003e.\u003c\/p\u003e\n\n\u003cp\u003eBut poking at \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/libvirt\/qemu\/i-00073770.xml\u003c\/tt\u003e on \u003ctt class=\"remarkup-monospaced\"\u003ecloudvirt1017\u003c\/tt\u003e (the host where the VM is running) those values seem to be applied only to \u003ctt class=\"remarkup-monospaced\"\u003e\/dev\/sda\u003c\/tt\u003e and not to \u003ctt class=\"remarkup-monospaced\"\u003e\/dev\/sdb\u003c\/tt\u003e:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e<target dev='sda' bus='scsi'\/>\n<iotune>\n  <total_bytes_sec>2000000000<\/total_bytes_sec>\n  <read_iops_sec>50000<\/read_iops_sec>\n  <write_iops_sec>5000<\/write_iops_sec>\n<\/iotune>\n\n<target dev='sdb' bus='scsi'\/>\n<iotune>\n  <total_bytes_sec>200000000<\/total_bytes_sec>\n  <write_iops_sec>500<\/write_iops_sec>\n<\/iotune>\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eNow I have to find a way to modify the values for \u003ctt class=\"remarkup-monospaced\"\u003esdb\u003c\/tt\u003e...\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_596\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wq3jawsqwgcxz76v25ba\/PHID-FILE-fmxl5u6fdfhz672tcxd7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/taavi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_594\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_595\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8395552\" id=\"8395552\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_593\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/taavi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_151\"\u003etaavi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8395552\" data-sigil=\"has-tooltip\" data-meta=\"0_592\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 11:28 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 11:28:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_590\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_591\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_152\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T301949#8395518\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949#8395518\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fnegri\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fnegri\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eNow I have to find a way to modify the values for \u003ctt class=\"remarkup-monospaced\"\u003esdb\u003c\/tt\u003e...\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI suspect this means we need a custom 'volume type' for the Cinder volume.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_605\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8395614\" id=\"8395614\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_604\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_153\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8395614\" data-sigil=\"has-tooltip\" data-meta=\"0_603\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 11:43 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 11:43:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_601\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_602\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_154\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eI suspect this means we need a custom 'volume type' for the Cinder volume.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eYup, I found the same thing, and it did work!\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@cloudcontrol1005:~# OS_PROJECT_ID=tools openstack volume qos create --property "read_iops_sec=20000" --property "write_iops_sec=10000" high-iops\nroot@cloudcontrol1005:~# OS_PROJECT_ID=tools openstack volume type create high-iops\nroot@cloudcontrol1005:~# OS_PROJECT_ID=tools openstack volume qos associate fc8ba52e-c079-4f56-b342-6f359366b08e 802972e4-71fa-4c34-a923-12e61307bddf\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI then stopped mariadb, umounted and unattached the volume, modified the volume type from Horizon, reattached, remounted, restarted mariadb.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@cloudvirt1017:~# virsh dumpxml i-00073770\n\n[...]\n\n      <target dev='sdb' bus='scsi'\/>\n      <iotune>\n        <read_iops_sec>20000<\/read_iops_sec>\n        <write_iops_sec>10000<\/write_iops_sec>\n      <\/iotune>\n\n[...]\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_615\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v2iza5xhmlrh6nzci6b6\/PHID-FILE-jtmoofc7k3odr7cyyhyu\/profile-martian.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Andrew\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8397462\" id=\"8397462\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_614\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_155\"\u003eAndrew\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8397462\" data-sigil=\"has-tooltip\" data-meta=\"0_613\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 7:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 19:42:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_611\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_612\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis is promising :)\u003c\/p\u003e\n\n\u003cp\u003eI'm noticing that the high-iops flavor is currently visible to all projects, and that the cli for modifying that seems broken (well, or maybe not broken, but intolerant of our project ids not being proper uuids.).  I'll hack in a fix for now and see if I can understand what's happening upstream.\u003c\/p\u003e\n\n\u003cp\u003e(edited to add: proposed upstream fix for this is \u003ca href=\"https:\/\/review.opendev.org\/c\/openstack\/cinder\/+\/864585\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/review.opendev.org\/c\/openstack\/cinder\/+\/864585\u003c\/a\u003e but I don't really expect it to be merged as it only barely counts as a 'fix')\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_624\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8397857\" id=\"8397857\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_623\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_157\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8397857\" data-sigil=\"has-tooltip\" data-meta=\"0_622\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 9:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 21:37:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_620\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_621\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_158\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 857073 had a related patch set uploaded (by Andrew Bogott; author: Andrew Bogott):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] Patch cinder volume_type api to allow non-uuid project ids.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/857073\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/857073\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_627\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8397858\" id=\"8397858\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_626\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_159\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_160\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8397858\" data-sigil=\"has-tooltip\" data-meta=\"0_625\"\u003e\u003cspan class=\"screen-only\"\u003eNov 15 2022, 9:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-15 21:37:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_636\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8398905\" id=\"8398905\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_635\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_161\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8398905\" data-sigil=\"has-tooltip\" data-meta=\"0_634\"\u003e\u003cspan class=\"screen-only\"\u003eNov 16 2022, 9:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-16 09:26:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_632\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_633\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSlight improvement in the replication lag graph after removing the IOPS limit, but I was hoping for something more:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/47jteq7tch22stml7mg4\/PHID-FILE-gdmzqlb6wnvqhll4nhyg\/Screenshot_2022-11-16_at_10.25.33.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_45\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/5gnvrievl4ig4ltuptes\/PHID-FILE-udfgv76cz7sbbjbtd5ks\/preview-Screenshot_2022-11-16_at_10.25.33.png\" width=\"220\" height=\"63.845223700121\" alt=\"Screenshot 2022-11-16 at 10.25.33.png (960\u00d73 px, 252 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eTrying to figure out what is the bottleneck now.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_646\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8406281\" id=\"8406281\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_645\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_163\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8406281\" data-sigil=\"has-tooltip\" data-meta=\"0_644\"\u003e\u003cspan class=\"screen-only\"\u003eNov 18 2022, 5:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-18 17:28:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_642\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_643\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_164\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe bottleneck is not IOPS now, because I can achieve much higher values with \u003ctt class=\"remarkup-monospaced\"\u003efio\u003c\/tt\u003e on the same disk:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@tools-db-2:~# fio --name=test_write --directory=$TEST_DIR --ioengine=libaio --rw=write\n[...]\n  write: IOPS=154k, BW=601MiB\/s\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThis is not representative of what mariadb is doing though, because mariadb is calling \u003ctt class=\"remarkup-monospaced\"\u003efsync\u003c\/tt\u003e after every commit. Adding \u003ctt class=\"remarkup-monospaced\"\u003e--fsync=1\u003c\/tt\u003e to the \u003ctt class=\"remarkup-monospaced\"\u003efio\u003c\/tt\u003e command yields:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@tools-db-2:~# fio --name=test_write --directory=$TEST_DIR --ioengine=libaio --rw=write --fsync=1\n[...]\n  write: IOPS=203, BW=814KiB\/s\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThis value seems to match roughly the number of \u003ctt class=\"remarkup-monospaced\"\u003eCOMMIT\u003c\/tt\u003e commands performed by mariadb on \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e (graph from yesterday):\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/tlkxfl2m2cl6pcmgms7s\/PHID-FILE-xr4djozbvooked7a3oxt\/Screenshot_2022-11-18_at_17.59.51.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_46\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/deqnjrouue54avd5ujkh\/PHID-FILE-cux6k2xecedd5qjjk646\/preview-Screenshot_2022-11-18_at_17.59.51.png\" width=\"220\" height=\"204.49023861171\" alt=\"Screenshot 2022-11-18 at 17.59.51.png (1\u00d71 px, 341 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eChanging the \u003ctt class=\"remarkup-monospaced\"\u003einnodb_flush_log_at_trx_commit\u003c\/tt\u003e option from \u003ctt class=\"remarkup-monospaced\"\u003e1\u003c\/tt\u003e to \u003ctt class=\"remarkup-monospaced\"\u003e2\u003c\/tt\u003e (as \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e suggested a few comments above) has a \u003cem\u003ehuge\u003c\/em\u003e impact (graph from today):\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/yj66b6uu4p5iqjeikq7o\/PHID-FILE-od7r4ybkd4evtpqprgkn\/Screenshot_2022-11-18_at_18.00.06.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_47\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/toaig7vum7uqfl434fan\/PHID-FILE-4aawoojrvhx27hpbtg55\/preview-Screenshot_2022-11-18_at_18.00.06.png\" width=\"220\" height=\"205.74514038877\" alt=\"Screenshot 2022-11-18 at 18.00.06.png (1\u00d71 px, 351 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eImmediately after changing the option there was a brief spike to 7k \u003ctt class=\"remarkup-monospaced\"\u003eCOMMIT\u003c\/tt\u003e per second, and mariadb quickly caught up with the replication lag, that went down from 10k seconds to zero very quickly:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/3tc6wcfcsp3vz6d2zws4\/PHID-FILE-h7iholjsg2eqyysumn46\/Screenshot_2022-11-18_at_18.18.06.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_48\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/2fv2o7rkot5gddm4caxi\/PHID-FILE-l4s6swnkmgr4wwdmf2bj\/preview-Screenshot_2022-11-18_at_18.18.06.png\" width=\"217.71889400922\" height=\"220\" alt=\"Screenshot 2022-11-18 at 18.18.06.png (1\u00d71 px, 302 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/6yezo5syrxsiczmo5zeh\/PHID-FILE-u7rzsi3wjeais5tvfphk\/Screenshot_2022-11-18_at_18.18.12.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_49\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/jn3avstl6z3k6wnby62g\/PHID-FILE-dpmlmtums7ldolwecmmr\/preview-Screenshot_2022-11-18_at_18.18.12.png\" width=\"218.47398843931\" height=\"220\" alt=\"Screenshot 2022-11-18 at 18.18.12.png (1\u00d71 px, 281 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eIn summary, I believe that if we are willing to accept the downside of setting \u003ctt class=\"remarkup-monospaced\"\u003einnodb_flush_log_at_trx_commit=2\u003c\/tt\u003e (namely, that \u003ca href=\"https:\/\/mariadb.com\/kb\/en\/innodb-system-variables\/#innodb_flush_log_at_trx_commit\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ea OS or power outage can cause the last second's transactions to be lost\u003c\/a\u003e), then our current Ceph setup is probably enough to migrate the entire database currently hosted at clouddb1001.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_655\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8406298\" id=\"8406298\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_654\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_165\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8406298\" data-sigil=\"has-tooltip\" data-meta=\"0_653\"\u003e\u003cspan class=\"screen-only\"\u003eNov 18 2022, 5:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-18 17:40:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_651\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_652\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_166\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOne question that I would like to answer before attempting a full migration is what are these big spikes in the number of transactions I see every night at 00:00 UTC on \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1001\u003c\/tt\u003e:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/krt5vz53f7k6eij6gheq\/PHID-FILE-e6d5yxj6d7thciyxohul\/Screenshot_2022-11-18_at_18.34.20.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_51\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/adlyuzlvryfk5jshwhds\/PHID-FILE-5g5tstc3ovzs5tqqise3\/preview-Screenshot_2022-11-18_at_18.34.20.png\" width=\"220\" height=\"216.26696832579\" alt=\"Screenshot 2022-11-18 at 18.34.20.png (1\u00d71 px, 315 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_658\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8420383\" id=\"8420383\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_657\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_167\"\u003efnegri\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-tpyjc4dqatdu6et\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8420383\" data-sigil=\"has-tooltip\" data-meta=\"0_656\"\u003e\u003cspan class=\"screen-only\"\u003eNov 24 2022, 4:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-24 16:58:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_667\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8425323\" id=\"8425323\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_666\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_168\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8425323\" data-sigil=\"has-tooltip\" data-meta=\"0_665\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2022, 5:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-28 17:05:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_663\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_664\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 857073 \u003cstrong\u003emerged\u003c\/strong\u003e by Andrew Bogott:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] Patch cinder volume_type api to allow non-uuid project ids.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/857073\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/857073\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_676\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8425389\" id=\"8425389\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_675\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_170\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8425389\" data-sigil=\"has-tooltip\" data-meta=\"0_674\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2022, 5:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-28 17:21:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_672\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_673\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 861430 had a related patch set uploaded (by Andrew Bogott; author: Andrew Bogott):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cinder: update volume_type_access.py.patch to resemble upstream patch\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/861430\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/861430\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_685\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8425408\" id=\"8425408\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_684\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_172\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8425408\" data-sigil=\"has-tooltip\" data-meta=\"0_683\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2022, 5:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-11-28 17:22:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_681\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_682\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_173\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 861430 \u003cstrong\u003emerged\u003c\/strong\u003e by Andrew Bogott:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] cinder: update volume_type_access.py.patch to resemble upstream patch\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/861430\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/861430\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_688\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8450907\" id=\"8450907\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_687\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_174\"\u003efnegri\u003c\/a\u003e added a subtask: \u003ca href=\"\/T324629\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_175\"\u003eT324629: toolsdb: What happens at Midnight?\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8450907\" data-sigil=\"has-tooltip\" data-meta=\"0_686\"\u003e\u003cspan class=\"screen-only\"\u003eDec 7 2022, 3:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-07 15:20:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_697\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8451663\" id=\"8451663\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_696\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_176\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8451663\" data-sigil=\"has-tooltip\" data-meta=\"0_695\"\u003e\u003cspan class=\"screen-only\"\u003eDec 7 2022, 6:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-07 18:05:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_693\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_694\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/8HnD7YQB6FQ6iqKiUee5\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"4_6\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-12-07T18:05:10Z] <wm-bot2> Increased quotas by 100 gigabytes (\u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e) - cookbook ran by fran@wmf3169\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_706\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8451893\" id=\"8451893\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_705\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_178\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8451893\" data-sigil=\"has-tooltip\" data-meta=\"0_704\"\u003e\u003cspan class=\"screen-only\"\u003eDec 7 2022, 7:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-07 19:18:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_702\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_703\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eQuick recap of the state of this ticket and planned next steps:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ewe want to migrate clouddb1001 (and its replica clouddb1002) to new Ceph-based VMs\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ewe believe that setting \u003ctt class=\"remarkup-monospaced\"\u003einnodb_flush_log_at_trx_commit=2\u003c\/tt\u003e the Ceph volume will be fast enough to sustain the current load, so splitting some of the big databases to separate hosts (as discussed in \u003ca href=\"\/T224154\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT224154\u003c\/span\u003e\u003c\/a\u003e) is not a strict requirement for this task\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ethere are weird spikes in the number of queries (\u003ca href=\"\/T324629\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT324629\u003c\/span\u003e\u003c\/a\u003e) but they don't cause a spike in IOPS so that shouldn't be a blocker either\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eNext steps:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMy favorite strategy to migrate would be to take a full local backup of clouddb1001 using \u003ca href=\"https:\/\/mariadb.com\/kb\/en\/setting-up-a-replica-with-mariabackup\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/mariadb.com\/kb\/en\/setting-up-a-replica-with-mariabackup\/\u003c\/a\u003e, that backup can be created without any downtime of the primary but it requires ~3TB of local space, which we don't have. My hope is that we manage to attach a Cinder volume to clouddb1001 (\u003ca href=\"\/T324700\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT324700\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e), save the backup there, then unattach the volume and reattach it to the new VM, which could then start replicating from clouddb1001.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFailing that, we could set up the replica using rsync, as was done in \u003ca href=\"\/T266587\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT266587\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, but that requires about 8-12 hours of downtime of the primary. Maybe setting the primary to read-only mode would be enough, instead of shutting it down completely (to be verified).\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_709\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v2iza5xhmlrh6nzci6b6\/PHID-FILE-jtmoofc7k3odr7cyyhyu\/profile-martian.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Andrew\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8453031\" id=\"8453031\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_708\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003eAndrew\u003c\/a\u003e closed subtask \u003ca href=\"\/T324700\" class=\"phui-handle handle-status-closed\"\u003eT324700: Attach a Cinder volume to clouddb1001\/1002\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8453031\" data-sigil=\"has-tooltip\" data-meta=\"0_707\"\u003e\u003cspan class=\"screen-only\"\u003eDec 8 2022, 3:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-08 03:11:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_712\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8456840\" id=\"8456840\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_711\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_181\"\u003efnegri\u003c\/a\u003e mentioned this in \u003ca href=\"\/T324846\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003eT324846: Reinstate innodb_large_prefix on ToolsDB (clouddb1001)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8456840\" data-sigil=\"has-tooltip\" data-meta=\"0_710\"\u003e\u003cspan class=\"screen-only\"\u003eDec 9 2022, 2:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-09 14:21:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_721\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8457335\" id=\"8457335\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_720\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_183\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8457335\" data-sigil=\"has-tooltip\" data-meta=\"0_719\"\u003e\u003cspan class=\"screen-only\"\u003eDec 9 2022, 5:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-09 17:20:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_717\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_718\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_184\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/21Hm94QB8Fs0LHO5yroZ\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"4_7\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud-feed)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2022-12-09T17:20:03Z] <wm-bot2> Increased quotas by 4000 gigabytes (\u003ca href=\"\/T301949\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT301949\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e) - cookbook ran by fran@wmf3169\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_730\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8457464\" id=\"8457464\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_729\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_185\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8457464\" data-sigil=\"has-tooltip\" data-meta=\"0_728\"\u003e\u003cspan class=\"screen-only\"\u003eDec 9 2022, 6:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-12-09 18:16:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_726\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_727\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_186\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI managed to create a new 4TB volume. As a test (before doing it on the primary clouddb1001) I've attached it to clouddb1002, formatted it to ext4, mounted it to \u003ctt class=\"remarkup-monospaced\"\u003e\/mnt\/tools-db\u003c\/tt\u003e and started a full backup from a Screen session:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clouddb1002:~# mariabackup --backup --target-dir=\/mnt\/tools-db --user root\n\n[...]\n\n2022-12-09 17:50:04 7f408ebc12c0  InnoDB: Operating system error number 24 in a file operation.\nInnoDB: Error number 24 means 'Too many open files'.\n\n[...]\n\nroot@clouddb1002:~# ulimit -a\n\n[...]\n\nopen files                      (-n) 1024\n\n[...]\n\nroot@clouddb1002:~# ulimit -n 10240\nroot@clouddb1002:~# mariabackup --backup --target-dir=\/mnt\/tools-db --user root\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI had to increase the \u003ctt class=\"remarkup-monospaced\"\u003eulimit\u003c\/tt\u003e for open files, and after that the backup started successfully and is currently in progress.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_739\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8499575\" id=\"8499575\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_738\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_187\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8499575\" data-sigil=\"has-tooltip\" data-meta=\"0_737\"\u003e\u003cspan class=\"screen-only\"\u003eJan 4 2023, 6:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-04 18:12:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_735\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_736\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_188\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eApologies for the lack of updates, I was busy with another project.\u003c\/p\u003e\n\n\u003cp\u003eThe test backup that I started on Dec 9 on the replica instance (\u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e) completed successfully, but looking at the output I realized that \u003ctt class=\"remarkup-monospaced\"\u003emariabackup\u003c\/tt\u003e does lock the tables while performing the backup (I was expecting it to lock only when running with the \u003ctt class=\"remarkup-monospaced\"\u003e--prepare\u003c\/tt\u003e option). This is not ideal because for the actual migration we will have to run it on the primary instance (because the replica instance is an incomplete replica), and that means any "read lock" will cause some downtime to the tools using the database. Some hours of downtime could be acceptable, but I'd like to better understand the expected locking behavior of mariabackup.\u003c\/p\u003e\n\n\u003cp\u003eThe \u003ca href=\"https:\/\/mariadb.com\/kb\/en\/how-mariabackup-works\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eofficial mariabackup docs\u003c\/a\u003e seem to imply that tables are locked at the start of the backup, and unlocked at the end, but I checked in Grafana and the replication lag remained fairly low during the backup, with only a few spikes, which makes me think that the database was in "read lock" only for a small time and not for the entire duration of the backup process. If anyone has more information on what is the expected locking behavior when running \u003ctt class=\"remarkup-monospaced\"\u003emariabackup\u003c\/tt\u003e against a primary instance, please let me know. :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_742\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8499619\" id=\"8499619\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_741\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_189\"\u003efnegri\u003c\/a\u003e mentioned this in \u003ca href=\"\/T326261\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_190\"\u003eT326261: [toolsdb] clouddb1002 stopped replicating from clouddb1001 (again)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8499619\" data-sigil=\"has-tooltip\" data-meta=\"0_740\"\u003e\u003cspan class=\"screen-only\"\u003eJan 4 2023, 6:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-04 18:23:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_755\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_751\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_752\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_753\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_754\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8499635\" id=\"8499635\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_750\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8499635\" data-sigil=\"has-tooltip\" data-meta=\"0_749\"\u003e\u003cspan class=\"screen-only\"\u003eJan 4 2023, 6:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-04 18:30:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_747\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_748\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_192\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eTables are locked (reads are fine) only when issuing the copy of the .frm files, which should be pretty fast.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_764\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8512211\" id=\"8512211\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_763\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_193\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8512211\" data-sigil=\"has-tooltip\" data-meta=\"0_762\"\u003e\u003cspan class=\"screen-only\"\u003eJan 10 2023, 11:21 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-10 11:21:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_760\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_761\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_194\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAfter resolving \u003ca href=\"\/T326261\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT326261\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, I started a new full backup from \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e to the attached Cinder volume:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clouddb1002:~# mariabackup --backup --target-dir=\/mnt\/tools-db\/backup-clouddb1002-20220110 --user root\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eWhen it completes, I will attempt to restore it on the new VM \u003ctt class=\"remarkup-monospaced\"\u003etools-db-2\u003c\/tt\u003e and see if it can chain-replicate successfully from \u003ctt class=\"remarkup-monospaced\"\u003eclouddb1002\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_773\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8513418\" id=\"8513418\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_772\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_195\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8513418\" data-sigil=\"has-tooltip\" data-meta=\"0_771\"\u003e\u003cspan class=\"screen-only\"\u003eJan 10 2023, 5:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-10 17:28:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_769\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_770\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe backup failed after about 15 minutes with the following error:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e230110 11:29:30 [01] Copying .\/s51138__heritage_p\/monuments_hr_@0028hr@0029.ibd to \/mnt\/tools-db\/backup-clouddb1002-20220110\/s51138__heritage_p\/monuments_hr_@0028hr@0029.ibd\n2023-01-10 11:29:30 140337901205248 [ERROR] InnoDB: Tried to read 10485760 bytes at offset 0. Was only able to read 131072.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI'm not sure why, I tried restarting it to see if it fails again. I was able to complete a full backup in December.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_786\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_782\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_783\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_784\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_785\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8514079\" id=\"8514079\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_781\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_197\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8514079\" data-sigil=\"has-tooltip\" data-meta=\"0_780\"\u003e\u003cspan class=\"screen-only\"\u003eJan 10 2023, 7:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-10 19:43:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_778\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_779\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI am not sure if that could be partial (or soon) some InnoDB tablespace corruption. I have refreshed the table stats and ran analyze and it worked fine. Can you retry?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_795\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8516021\" id=\"8516021\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_794\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_199\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8516021\" data-sigil=\"has-tooltip\" data-meta=\"0_793\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2023, 10:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-11 10:16:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_791\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_792\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_200\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFailed again but this time after running for 2 hours, with 1.8T backed up (about 50% of the data):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e230110 20:35:34 [01] Copying .\/s52861__secWatch\/temp_pagelinks.ibd to \/mnt\/tools-db\/backup-clouddb1002-20220110\/s52861__secWatch\/temp_pagelinks.ibd\n2023-01-10 20:35:34 139950603368192 [ERROR] InnoDB: Tried to read 147456 bytes at offset 0. Was only able to read 114688\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eRetrying one more time, if it fails again I think I will try backing up from the primary instead.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_808\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_804\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_805\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_806\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_807\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8516466\" id=\"8516466\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_803\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_201\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8516466\" data-sigil=\"has-tooltip\" data-meta=\"0_802\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2023, 1:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-11 13:33:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_800\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_801\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat's also a different table too, right?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_817\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8516730\" id=\"8516730\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_816\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_203\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8516730\" data-sigil=\"has-tooltip\" data-meta=\"0_815\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2023, 3:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-11 15:28:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_813\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_814\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_204\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThird attempt, third failure:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e230111 10:31:00 [01] Copying .\/s51138__heritage_p\/monuments_by_@0028be@002dtarask@0029.ibd to \/mnt\/tools-db\/backup-clouddb1002-20220110\/s51138__heritage_p\/monuments_by_@0028be@002dtarask@0029.ibd\n2023-01-11 10:31:00 140383593953024 [ERROR] InnoDB: Tried to read 10485760 bytes at offset 0. Was only able to read 131072.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAssuming the error is on the table indicated by the last log line before the error, the tables are indeed different ones.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_830\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_826\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_827\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_828\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_829\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8516735\" id=\"8516735\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_825\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_205\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8516735\" data-sigil=\"has-tooltip\" data-meta=\"0_824\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2023, 3:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-11 15:30:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_822\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_823\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_206\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis smells like InnoDB tablespace corruption to me\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_839\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8516771\" id=\"8516771\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_838\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_207\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8516771\" data-sigil=\"has-tooltip\" data-meta=\"0_837\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2023, 3:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-11 15:39:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_835\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_836\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_208\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAgreed, hopefully they're corrupted only in the replica. I'm trying to backup only a single database (\u003ctt class=\"remarkup-monospaced\"\u003es51138__heritage_p\u003c\/tt\u003e) to see if I can reproduce the issue.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_848\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8516853\" id=\"8516853\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_847\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_209\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8516853\" data-sigil=\"has-tooltip\" data-meta=\"0_846\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2023, 3:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-11 15:57:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_844\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_845\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_210\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eBacking up the single database worked 3 times in a row without errors \u00af\\_(\u30c4)_\/\u00af\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_858\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8523889\" id=\"8523889\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_857\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_211\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8523889\" data-sigil=\"has-tooltip\" data-meta=\"0_856\"\u003e\u003cspan class=\"screen-only\"\u003eJan 13 2023, 3:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-13 15:19:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_854\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_855\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_212\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have attached the Cinder volume to the primary (clouddb1001) and started a backup there. I'm keeping an eye on \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/000000273\/tools-mariadb\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eGrafana\u003c\/a\u003e to see if the impact on db performance is acceptable.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clouddb1001:~# ulimit -n 10240\nroot@clouddb1001:~# mariabackup --backup --target-dir=\/mnt\/tools-db\/backup-clouddb1001-20220113 --user root\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_867\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8524210\" id=\"8524210\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_866\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_213\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8524210\" data-sigil=\"has-tooltip\" data-meta=\"0_865\"\u003e\u003cspan class=\"screen-only\"\u003eJan 13 2023, 4:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-13 16:52:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_863\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_864\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_214\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe impact and performance seems minor, disk throughput (read) increased a lot as expected, disk latency (read) increased slightly, but query latency was stable.\u003c\/p\u003e\n\n\u003cp\u003eAfter about 90 minutes, 872GB out of 2726GB were backed up (30%). I have now stopped the backup to avoid any unexpected issues during the weekend. I will start it again from scratch on Monday morning around 9:00 UTC so I can monitor it during the day, and try to restore it on the same day or the day after.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_876\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8527242\" id=\"8527242\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_875\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_215\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8527242\" data-sigil=\"has-tooltip\" data-meta=\"0_874\"\u003e\u003cspan class=\"screen-only\"\u003eJan 16 2023, 9:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-16 09:14:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_872\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_873\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_216\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI executed this command at 9:12 UTC and it's currently running in a tmux session in clouddb1001:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clouddb1001:~# mariabackup --backup --target-dir=\/mnt\/tools-db\/backup-clouddb1001-20230116 --user root\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_885\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8527645\" id=\"8527645\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_884\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_217\"\u003efnegri\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_218\"\u003ejcrespo\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8527645\" data-sigil=\"has-tooltip\" data-meta=\"0_883\"\u003e\u003cspan class=\"screen-only\"\u003eJan 16 2023, 12:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-16 12:20:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_881\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_882\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_219\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe backup is currently at about 50% (1.7T written).\u003c\/p\u003e\n\n\u003cp\u003eI discussed this in #wikidata-data-persistence and \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e told me it's impossible to predict how long the read-only lock will last during the last phase of the backup. I will keep an eye on it and cancel the backup if the read-only lock lasts for longer than 30 minutes. I have also sent out an email to \u003ctt class=\"remarkup-monospaced\"\u003ecloud@lists.wikimedia.org\u003c\/tt\u003e to inform Tools maintainers of potential disruption.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_894\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dukwhflkzovmqspqebzq\/PHID-FILE-myl66zkneqzqret7f7dq\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fnegri\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8527836\" id=\"8527836\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_893\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fnegri\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_220\"\u003efnegri\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8527836\" data-sigil=\"has-tooltip\" data-meta=\"0_892\"\u003e\u003cspan class=\"screen-only\"\u003eJan 16 2023, 2:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2023-01-16 14:17:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_890\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_891\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_221\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe backup failed at about 85% with the following error:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e230116 13:23:53 [01] Copying .\/s51290__dpl_p\/ch_fixed_links.ibd to \/mnt\/tools-db\/backup-clouddb1001-20230116\/s51290__dpl_p\/ch_fixed_links.ibd\n2023-01-16 13:23:53 140452300846848 [ERROR] InnoDB: Tried to read 294912 bytes at offset 0. Was only able to read 278528.\n[01] mariabackup: Error: xtrabackup_copy_datafile() failed.\n[01] mariabackup: Error: failed to copy datafile.\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-pezzwicpjfd666axk7si"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vvww3faq62q2b7j2nfxb","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-d6du6n6q4ie2b77nrvj2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-76qt3bhv6eefo74agkbc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vvww3faq62q2b7j2nfxb","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-76qt3bhv6eefo74agkbc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-76qt3bhv6eefo74agkbc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-icgyxxnpr43dhzjvrsb2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-szg74fzvajhhmh5rolur"}},{"phid":"PHID-FILE-r3dejxze7aumfqa3ln46","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/ermtvjirnbsazzrj6qgw\/PHID-FILE-r3dejxze7aumfqa3ln46\/Screenshot_2022-11-11_at_23.09.20.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/ermtvjirnbsazzrj6qgw\/PHID-FILE-r3dejxze7aumfqa3ln46\/Screenshot_2022-11-11_at_23.09.20.png","alt":"Screenshot 2022-11-11 at 23.09.20.png (1\u00d71 px, 652 KB)","viewable":true,"monogram":"F35756562"},{"phid":"PHID-FILE-hszernfidfsklhzxkrem","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/lzrt7677v2nsx5jkerll\/PHID-FILE-hszernfidfsklhzxkrem\/Screenshot_2022-11-11_at_23.09.40.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/lzrt7677v2nsx5jkerll\/PHID-FILE-hszernfidfsklhzxkrem\/Screenshot_2022-11-11_at_23.09.40.png","alt":"Screenshot 2022-11-11 at 23.09.40.png (1\u00d71 px, 521 KB)","viewable":true,"monogram":"F35756561"},{"phid":"PHID-FILE-z2jedi7oox7w3n66hoje","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/hdo63p7y7v7vfbrdhlam\/PHID-FILE-z2jedi7oox7w3n66hoje\/Screenshot_2022-11-12_at_22.30.00.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/hdo63p7y7v7vfbrdhlam\/PHID-FILE-z2jedi7oox7w3n66hoje\/Screenshot_2022-11-12_at_22.30.00.png","alt":"Screenshot 2022-11-12 at 22.30.00.png (1\u00d71 px, 247 KB)","viewable":true,"monogram":"F35763485"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"phid":"PHID-FILE-cfmqff4ajcmnmut3govv","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/bgmn5kfpdx362vkhipap\/PHID-FILE-cfmqff4ajcmnmut3govv\/Screenshot_2022-11-14_at_10.49.14.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/bgmn5kfpdx362vkhipap\/PHID-FILE-cfmqff4ajcmnmut3govv\/Screenshot_2022-11-14_at_10.49.14.png","alt":"Screenshot 2022-11-14 at 10.49.14.png (844\u00d71 px, 282 KB)","viewable":true,"monogram":"F35773584"},{"phid":"PHID-FILE-4pbdwzvuc2xr4hvudql4","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/cptxioonrxhlda2jzo7a\/PHID-FILE-4pbdwzvuc2xr4hvudql4\/Screenshot_2022-11-14_at_21.31.32.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/cptxioonrxhlda2jzo7a\/PHID-FILE-4pbdwzvuc2xr4hvudql4\/Screenshot_2022-11-14_at_21.31.32.png","alt":"Screenshot 2022-11-14 at 21.31.32.png (762\u00d71 px, 577 KB)","viewable":true,"monogram":"F35777643"},{"phid":"PHID-FILE-ahjk463wc3pnhmcs7wix","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/33xouc5mi7zwpux2ffnq\/PHID-FILE-ahjk463wc3pnhmcs7wix\/Screenshot_2022-11-14_at_21.32.16.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/33xouc5mi7zwpux2ffnq\/PHID-FILE-ahjk463wc3pnhmcs7wix\/Screenshot_2022-11-14_at_21.32.16.png","alt":"Screenshot 2022-11-14 at 21.32.16.png (818\u00d71 px, 150 KB)","viewable":true,"monogram":"F35777648"},{"phid":"PHID-FILE-ag7hkupi4htfx5lj24ky","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/s5ol5lnccl7l7l7j4upg\/PHID-FILE-ag7hkupi4htfx5lj24ky\/Screenshot_2022-11-14_at_21.32.48.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/s5ol5lnccl7l7l7j4upg\/PHID-FILE-ag7hkupi4htfx5lj24ky\/Screenshot_2022-11-14_at_21.32.48.png","alt":"Screenshot 2022-11-14 at 21.32.48.png (948\u00d71 px, 586 KB)","viewable":true,"monogram":"F35777654"},{"phid":"PHID-FILE-73xmazrrh2optmevbafi","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/u4wx6flecr25hnbp6hb7\/PHID-FILE-73xmazrrh2optmevbafi\/Screenshot_2022-11-15_at_11.41.46.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/u4wx6flecr25hnbp6hb7\/PHID-FILE-73xmazrrh2optmevbafi\/Screenshot_2022-11-15_at_11.41.46.png","alt":"Screenshot 2022-11-15 at 11.41.46.png (946\u00d71 px, 185 KB)","viewable":true,"monogram":"F35780867"},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"phid":"PHID-FILE-gdmzqlb6wnvqhll4nhyg","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/47jteq7tch22stml7mg4\/PHID-FILE-gdmzqlb6wnvqhll4nhyg\/Screenshot_2022-11-16_at_10.25.33.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/47jteq7tch22stml7mg4\/PHID-FILE-gdmzqlb6wnvqhll4nhyg\/Screenshot_2022-11-16_at_10.25.33.png","alt":"Screenshot 2022-11-16 at 10.25.33.png (960\u00d73 px, 252 KB)","viewable":true,"monogram":"F35785268"},{"phid":"PHID-FILE-xr4djozbvooked7a3oxt","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/tlkxfl2m2cl6pcmgms7s\/PHID-FILE-xr4djozbvooked7a3oxt\/Screenshot_2022-11-18_at_17.59.51.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/tlkxfl2m2cl6pcmgms7s\/PHID-FILE-xr4djozbvooked7a3oxt\/Screenshot_2022-11-18_at_17.59.51.png","alt":"Screenshot 2022-11-18 at 17.59.51.png (1\u00d71 px, 341 KB)","viewable":true,"monogram":"F35795260"},{"phid":"PHID-FILE-od7r4ybkd4evtpqprgkn","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/yj66b6uu4p5iqjeikq7o\/PHID-FILE-od7r4ybkd4evtpqprgkn\/Screenshot_2022-11-18_at_18.00.06.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/yj66b6uu4p5iqjeikq7o\/PHID-FILE-od7r4ybkd4evtpqprgkn\/Screenshot_2022-11-18_at_18.00.06.png","alt":"Screenshot 2022-11-18 at 18.00.06.png (1\u00d71 px, 351 KB)","viewable":true,"monogram":"F35795272"},{"phid":"PHID-FILE-h7iholjsg2eqyysumn46","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/3tc6wcfcsp3vz6d2zws4\/PHID-FILE-h7iholjsg2eqyysumn46\/Screenshot_2022-11-18_at_18.18.06.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/3tc6wcfcsp3vz6d2zws4\/PHID-FILE-h7iholjsg2eqyysumn46\/Screenshot_2022-11-18_at_18.18.06.png","alt":"Screenshot 2022-11-18 at 18.18.06.png (1\u00d71 px, 302 KB)","viewable":true,"monogram":"F35795292"},{"phid":"PHID-FILE-u7rzsi3wjeais5tvfphk","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/6yezo5syrxsiczmo5zeh\/PHID-FILE-u7rzsi3wjeais5tvfphk\/Screenshot_2022-11-18_at_18.18.12.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/6yezo5syrxsiczmo5zeh\/PHID-FILE-u7rzsi3wjeais5tvfphk\/Screenshot_2022-11-18_at_18.18.12.png","alt":"Screenshot 2022-11-18 at 18.18.12.png (1\u00d71 px, 281 KB)","viewable":true,"monogram":"F35795294"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"phid":"PHID-FILE-e6d5yxj6d7thciyxohul","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/krt5vz53f7k6eij6gheq\/PHID-FILE-e6d5yxj6d7thciyxohul\/Screenshot_2022-11-18_at_18.34.20.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/krt5vz53f7k6eij6gheq\/PHID-FILE-e6d5yxj6d7thciyxohul\/Screenshot_2022-11-18_at_18.34.20.png","alt":"Screenshot 2022-11-18 at 18.34.20.png (1\u00d71 px, 315 KB)","viewable":true,"monogram":"F35795342"},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-icgyxxnpr43dhzjvrsb2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-y7dabewmrzhzrzyl3mdj"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-thfaocargcuwssb7l2oe"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-pezzwicpjfd666axk7si"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ooq6isvbnp7wjgvjl7ue"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-sjaibs3bcs5nstmutlag"}},{"phid":"PHID-XACT-TASK-h34xvr4rrqamt76"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-hvejlvjb6yhsmiwcb6ny"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-hvejlvjb6yhsmiwcb6ny"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-pv6orh344v36ncs"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-2e742cgxue5s57d"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-4je6dcufnyyoua5"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"phid":"PHID-XACT-TASK-w76r3v4qolpuyh2"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-t3ou43epljylypr33c7o"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-sr6sin4gusccdtha5zx5"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-m4445fdl2ov5zugq7nbt"}},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu"}},{"phid":"PHID-XACT-TASK-36g4o643qevtna5"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu"}},{"phid":"PHID-XACT-TASK-gbwxlamghtv2ddb"},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu"}},{"phid":"PHID-XACT-TASK-2uw55fd5epay7z7"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"phid":"PHID-XACT-TASK-fldvjzoltwnbrty"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"phid":"PHID-XACT-TASK-dz2wpcovraw6fmp"},{"hovercardSpec":{"objectPHID":"PHID-USER-bfjltur45gvx4i723lyk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vvww3faq62q2b7j2nfxb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-3nkvk2rmtbbiyt3"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"phid":"PHID-XACT-TASK-d4jtcqbdrvo5dju"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-4kek3pz32qx5274"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-nz36uats44iiepw"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-tn2ne5zrmzgbwfw"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-h3hb7wqu7w3h42l"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-d6du6n6q4ie2b77nrvj2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-uzorenext6ta4lf"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-b5kmct72nrozpzx"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-adjluu67ttkjxqn"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-mck6tdblbnbp6os"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bvud2nsvebx5bzor33ya"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bftyx7dc3hykv3kfdje6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-7sul7hj32btacinsnubq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-7ebw6u6e2z5qpqb"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-vsglllkn4rrv4f6"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-tqom3qnhss5xalw"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-nbse5ilfocwvdwm"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-fzzpizxysmncagl"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-rmsrkum26pm2b4y"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-lvgzvuwyhtlze4s"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-ksr4r4mooiv37tr"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-5rpgmj7axbspsni"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-2u2sgmj6vq4cufd"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-lxseh6kjs5hxbhr"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-tosvpbtdarvcpkh"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-a72fnh2oyeeyctr"},{"hovercardSpec":{"objectPHID":"PHID-USER-mqhfa5y5f5tg6g3b5hui"}},{"phid":"PHID-XACT-TASK-klo4u75z5d2zabr"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-qvfm2h6vqq42lte"},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu"}},{"phid":"PHID-XACT-TASK-3jtntoulu4mlvt6"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-zlibfcwco3ie7rt"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-lfp3a2yuuragrem"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-3q2yngp6ku6ww44"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-fhlajs3dpodvfv7"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-n4xpd77jvxjvlru"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-yxx5jdsfgdnuze5"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-2w7b4xrhm227k4c"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-y7dabewmrzhzrzyl3mdj"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-dg4krp4y6ehnihs"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-cc5i55uhntxwjx6"},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-gac3djiqa5aizx6tilz3"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-7byxwpfgmkhhbzz"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-csn3ri7mbvkbuzp"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-tc7btckmwlqpnll"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ooq6isvbnp7wjgvjl7ue"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-y4e5y2dkyauc5yl"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-537kj3hb2iawfwy"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-rxt56fckkhnyqki"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-aq3myonx3iojsze"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-jzq7r6o2hbzxjou"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-sos4vlldvv2kf2h"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-tdwqaafp5sqdkop"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-vjdr4rjs7hlaxao"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-xvobkat7koslhje"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-mjx3vldczieacl5"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-bxmiucjcowojb6r"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-anaxudhz66gjhfv"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-lw6s5scexefzizm"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-pocj5c5bsotxv6z"},{"hovercardSpec":{"objectPHID":"PHID-USER-mhjb2hqevs3ygaiis3im"}},{"phid":"PHID-XACT-TASK-mszgqxlmvzoga7t"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-h34xvr4rrqamt76\/","ref":"T301949#8032927"},[],{"anchor":"8032927"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_222\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-h34xvr4rrqamt76\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_224\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-h34xvr4rrqamt76","anchor":"8032927"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-adg5tshjpwesnkp","anchor":"8033000"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3ci2kssq6iaxe3a","anchor":"8033732"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pv6orh344v36ncs\/","ref":"T301949#8033799"},[],{"anchor":"8033799"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_238\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pv6orh344v36ncs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_240\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pv6orh344v36ncs","anchor":"8033799"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-twpjdbn2jhyx7fv","anchor":"8033800"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2e742cgxue5s57d\/","ref":"T301949#8033827"},[],{"anchor":"8033827"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_250\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2e742cgxue5s57d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2e742cgxue5s57d","anchor":"8033827"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-g7q7t4cb4urmfa2","anchor":"8033866"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4je6dcufnyyoua5\/","ref":"T301949#8033872"},[],{"anchor":"8033872"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_262\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_263\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4je6dcufnyyoua5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_264\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4je6dcufnyyoua5","anchor":"8033872"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rnrkkfzyft4gdj3","anchor":"8033890"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-w76r3v4qolpuyh2\/","ref":"T301949#8035850"},[],{"anchor":"8035850"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_274\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_275\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-w76r3v4qolpuyh2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_276\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_277\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-w76r3v4qolpuyh2","anchor":"8035850"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z53fguecrtjejea","anchor":"8143314"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-36g4o643qevtna5\/","ref":"T301949#8153548"},[],{"anchor":"8153548"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_288\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-36g4o643qevtna5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_290\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-36g4o643qevtna5","anchor":"8153548"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hafarjsfxodoe6x","anchor":"8185781"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gbwxlamghtv2ddb\/","ref":"T301949#8262117"},[],{"anchor":"8262117"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_300\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gbwxlamghtv2ddb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_302\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gbwxlamghtv2ddb","anchor":"8262117"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2uw55fd5epay7z7\/","ref":"T301949#8262123"},[],{"anchor":"8262123"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_309\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2uw55fd5epay7z7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_311\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2uw55fd5epay7z7","anchor":"8262123"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fldvjzoltwnbrty\/","ref":"T301949#8266105"},[],{"anchor":"8266105"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_318\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fldvjzoltwnbrty\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_320\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-fldvjzoltwnbrty","anchor":"8266105"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dz2wpcovraw6fmp\/","ref":"T301949#8266106"},[],{"anchor":"8266106"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_329\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dz2wpcovraw6fmp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_331\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-dz2wpcovraw6fmp","anchor":"8266106"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4slbgfdo6pccfd5","anchor":"8266203"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uulp2ztsnjom3y6","anchor":"8287551"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3nkvk2rmtbbiyt3\/","ref":"T301949#8314788"},[],{"anchor":"8314788"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_346\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_347\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3nkvk2rmtbbiyt3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_348\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_349\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3nkvk2rmtbbiyt3","anchor":"8314788"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-d4jtcqbdrvo5dju\/","ref":"T301949#8314988"},[],{"anchor":"8314988"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_355\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-d4jtcqbdrvo5dju\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_357\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_358\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-d4jtcqbdrvo5dju","anchor":"8314988"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4kek3pz32qx5274\/","ref":"T301949#8317377"},[],{"anchor":"8317377"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_366\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_367\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4kek3pz32qx5274\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_368\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_369\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4kek3pz32qx5274","anchor":"8317377"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nz36uats44iiepw\/","ref":"T301949#8326055"},[],{"anchor":"8326055"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_375\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_376\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nz36uats44iiepw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_377\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_378\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nz36uats44iiepw","anchor":"8326055"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tn2ne5zrmzgbwfw\/","ref":"T301949#8328457"},[],{"anchor":"8328457"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_384\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_385\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tn2ne5zrmzgbwfw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_386\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_387\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tn2ne5zrmzgbwfw","anchor":"8328457"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nrugfbv7gm4erfa","anchor":"8338540"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-h3hb7wqu7w3h42l\/","ref":"T301949#8345255"},[],{"anchor":"8345255"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_396\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_397\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-h3hb7wqu7w3h42l\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_398\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-h3hb7wqu7w3h42l","anchor":"8345255"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-euytpl7gdzbs5i3","anchor":"8350470"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uzorenext6ta4lf\/","ref":"T301949#8350981"},[],{"anchor":"8350981"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_408\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_409\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uzorenext6ta4lf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_410\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_411\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3iystyu43d3o7br","anchor":"8350981"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-b5kmct72nrozpzx\/","ref":"T301949#8352177"},[],{"anchor":"8352177"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_417\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_418\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-b5kmct72nrozpzx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_419\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_420\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-b5kmct72nrozpzx","anchor":"8352177"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-adjluu67ttkjxqn\/","ref":"T301949#8353450"},[],{"anchor":"8353450"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_430\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-adjluu67ttkjxqn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_432\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-adjluu67ttkjxqn","anchor":"8353450"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-iwluh3iwqmmi3aa","anchor":"8380161"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mck6tdblbnbp6os\/","ref":"T301949#8383081"},[],{"anchor":"8383081"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_444\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_445\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mck6tdblbnbp6os\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_446\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_447\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mck6tdblbnbp6os","anchor":"8383081"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ribmcbyz5kxvwrf","anchor":"8383210"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7ebw6u6e2z5qpqb\/","ref":"T301949#8389718"},[],{"anchor":"8389718"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_456\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_457\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7ebw6u6e2z5qpqb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_458\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_459\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7ebw6u6e2z5qpqb","anchor":"8389718"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vsglllkn4rrv4f6\/","ref":"T301949#8389719"},[],{"anchor":"8389719"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_465\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_466\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vsglllkn4rrv4f6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_467\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_468\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vsglllkn4rrv4f6","anchor":"8389719"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tqom3qnhss5xalw\/","ref":"T301949#8389909"},[],{"anchor":"8389909"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_474\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_475\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tqom3qnhss5xalw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_476\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_477\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-tqom3qnhss5xalw","anchor":"8389909"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nbse5ilfocwvdwm\/","ref":"T301949#8390250"},[],{"anchor":"8390250"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_483\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_484\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nbse5ilfocwvdwm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_485\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_486\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-nbse5ilfocwvdwm\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_487\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nbse5ilfocwvdwm","anchor":"8390250"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fzzpizxysmncagl\/","ref":"T301949#8390668"},[],{"anchor":"8390668"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_493\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_494\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fzzpizxysmncagl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_495\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_496\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fzzpizxysmncagl","anchor":"8390668"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rmsrkum26pm2b4y\/","ref":"T301949#8391199"},[],{"anchor":"8391199"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_502\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_503\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rmsrkum26pm2b4y\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_504\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_505\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rmsrkum26pm2b4y","anchor":"8391199"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lvgzvuwyhtlze4s\/","ref":"T301949#8391522"},[],{"anchor":"8391522"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_515\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_516\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lvgzvuwyhtlze4s\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_517\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_518\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lvgzvuwyhtlze4s","anchor":"8391522"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ksr4r4mooiv37tr\/","ref":"T301949#8391524"},[],{"anchor":"8391524"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_524\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_525\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ksr4r4mooiv37tr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_526\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_527\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ksr4r4mooiv37tr","anchor":"8391524"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5rpgmj7axbspsni\/","ref":"T301949#8394082"},[],{"anchor":"8394082"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_533\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_534\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5rpgmj7axbspsni\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_535\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5rpgmj7axbspsni","anchor":"8394082"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2u2sgmj6vq4cufd\/","ref":"T301949#8395173"},[],{"anchor":"8395173"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_542\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_543\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2u2sgmj6vq4cufd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_544\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_545\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-2u2sgmj6vq4cufd","anchor":"8395173"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lxseh6kjs5hxbhr\/","ref":"T301949#8395192"},[],{"anchor":"8395192"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_555\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_556\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lxseh6kjs5hxbhr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_557\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_558\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-lxseh6kjs5hxbhr","anchor":"8395192"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tosvpbtdarvcpkh\/","ref":"T301949#8395372"},[],{"anchor":"8395372"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_568\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_569\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tosvpbtdarvcpkh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_570\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_571\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tosvpbtdarvcpkh","anchor":"8395372"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-a72fnh2oyeeyctr\/","ref":"T301949#8395518"},[],{"anchor":"8395518"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_577\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_578\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-a72fnh2oyeeyctr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_579\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_580\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-a72fnh2oyeeyctr","anchor":"8395518"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-klo4u75z5d2zabr\/","ref":"T301949#8395552"},[],{"anchor":"8395552"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_586\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_587\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-klo4u75z5d2zabr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_588\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_589\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-klo4u75z5d2zabr","anchor":"8395552"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qvfm2h6vqq42lte\/","ref":"T301949#8395614"},[],{"anchor":"8395614"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_597\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_598\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qvfm2h6vqq42lte\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_599\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_600\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qvfm2h6vqq42lte","anchor":"8395614"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3jtntoulu4mlvt6\/","ref":"T301949#8397462"},[],{"anchor":"8397462"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_606\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_607\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3jtntoulu4mlvt6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_608\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_609\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3jtntoulu4mlvt6\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_610\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3jtntoulu4mlvt6","anchor":"8397462"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zlibfcwco3ie7rt\/","ref":"T301949#8397857"},[],{"anchor":"8397857"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_616\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_617\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zlibfcwco3ie7rt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_618\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_619\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zlibfcwco3ie7rt","anchor":"8397857"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4atjfmgufndcggs","anchor":"8397858"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lfp3a2yuuragrem\/","ref":"T301949#8398905"},[],{"anchor":"8398905"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_628\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_629\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lfp3a2yuuragrem\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_630\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_631\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lfp3a2yuuragrem","anchor":"8398905"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3q2yngp6ku6ww44\/","ref":"T301949#8406281"},[],{"anchor":"8406281"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_637\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_638\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3q2yngp6ku6ww44\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_639\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_640\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3q2yngp6ku6ww44\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_641\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3q2yngp6ku6ww44","anchor":"8406281"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fhlajs3dpodvfv7\/","ref":"T301949#8406298"},[],{"anchor":"8406298"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_647\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_648\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fhlajs3dpodvfv7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_649\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_650\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fhlajs3dpodvfv7","anchor":"8406298"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tpyjc4dqatdu6et","anchor":"8420383"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-n4xpd77jvxjvlru\/","ref":"T301949#8425323"},[],{"anchor":"8425323"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_659\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_660\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-n4xpd77jvxjvlru\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_661\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_662\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-n4xpd77jvxjvlru","anchor":"8425323"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yxx5jdsfgdnuze5\/","ref":"T301949#8425389"},[],{"anchor":"8425389"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_668\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_669\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yxx5jdsfgdnuze5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_670\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_671\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-yxx5jdsfgdnuze5","anchor":"8425389"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2w7b4xrhm227k4c\/","ref":"T301949#8425408"},[],{"anchor":"8425408"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_677\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2w7b4xrhm227k4c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_679\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_680\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2w7b4xrhm227k4c","anchor":"8425408"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kxse27xntshbid6","anchor":"8450907"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dg4krp4y6ehnihs\/","ref":"T301949#8451663"},[],{"anchor":"8451663"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_689\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_690\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dg4krp4y6ehnihs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_691\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_692\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dg4krp4y6ehnihs","anchor":"8451663"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cc5i55uhntxwjx6\/","ref":"T301949#8451893"},[],{"anchor":"8451893"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_698\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_699\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cc5i55uhntxwjx6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_700\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_701\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cc5i55uhntxwjx6","anchor":"8451893"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3juqkkdt7e7pty6","anchor":"8453031"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2fitxet3fb3djx7","anchor":"8456840"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7byxwpfgmkhhbzz\/","ref":"T301949#8457335"},[],{"anchor":"8457335"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_713\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7byxwpfgmkhhbzz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_715\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_716\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7byxwpfgmkhhbzz","anchor":"8457335"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-csn3ri7mbvkbuzp\/","ref":"T301949#8457464"},[],{"anchor":"8457464"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_722\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_723\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-csn3ri7mbvkbuzp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_724\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_725\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-csn3ri7mbvkbuzp","anchor":"8457464"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tc7btckmwlqpnll\/","ref":"T301949#8499575"},[],{"anchor":"8499575"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_731\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_732\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tc7btckmwlqpnll\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_733\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_734\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tc7btckmwlqpnll","anchor":"8499575"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-agbkikywkx57jwh","anchor":"8499619"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-y4e5y2dkyauc5yl\/","ref":"T301949#8499635"},[],{"anchor":"8499635"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_743\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_744\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-y4e5y2dkyauc5yl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_745\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_746\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-y4e5y2dkyauc5yl","anchor":"8499635"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-537kj3hb2iawfwy\/","ref":"T301949#8512211"},[],{"anchor":"8512211"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_756\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_757\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-537kj3hb2iawfwy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_758\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_759\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-537kj3hb2iawfwy","anchor":"8512211"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rxt56fckkhnyqki\/","ref":"T301949#8513418"},[],{"anchor":"8513418"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_765\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_766\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rxt56fckkhnyqki\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_767\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_768\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rxt56fckkhnyqki","anchor":"8513418"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-aq3myonx3iojsze\/","ref":"T301949#8514079"},[],{"anchor":"8514079"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_774\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_775\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aq3myonx3iojsze\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_776\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_777\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-aq3myonx3iojsze","anchor":"8514079"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jzq7r6o2hbzxjou\/","ref":"T301949#8516021"},[],{"anchor":"8516021"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_215\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_787\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_788\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_217\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jzq7r6o2hbzxjou\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_789\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_790\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jzq7r6o2hbzxjou","anchor":"8516021"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-sos4vlldvv2kf2h\/","ref":"T301949#8516466"},[],{"anchor":"8516466"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_219\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_796\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_797\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_221\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sos4vlldvv2kf2h\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_798\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_799\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-sos4vlldvv2kf2h","anchor":"8516466"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tdwqaafp5sqdkop\/","ref":"T301949#8516730"},[],{"anchor":"8516730"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_223\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_809\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_810\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_225\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tdwqaafp5sqdkop\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_811\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_812\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tdwqaafp5sqdkop","anchor":"8516730"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vjdr4rjs7hlaxao\/","ref":"T301949#8516735"},[],{"anchor":"8516735"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_227\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_818\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_819\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_229\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vjdr4rjs7hlaxao\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_820\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_821\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-vjdr4rjs7hlaxao","anchor":"8516735"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xvobkat7koslhje\/","ref":"T301949#8516771"},[],{"anchor":"8516771"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_231\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_831\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_832\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_233\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xvobkat7koslhje\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_833\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_834\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xvobkat7koslhje","anchor":"8516771"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mjx3vldczieacl5\/","ref":"T301949#8516853"},[],{"anchor":"8516853"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_235\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_840\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_841\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_237\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mjx3vldczieacl5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_842\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_843\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mjx3vldczieacl5","anchor":"8516853"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bxmiucjcowojb6r\/","ref":"T301949#8523889"},[],{"anchor":"8523889"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_239\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_849\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_850\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_241\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bxmiucjcowojb6r\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_851\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_852\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_243\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-bxmiucjcowojb6r\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_853\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bxmiucjcowojb6r","anchor":"8523889"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-anaxudhz66gjhfv\/","ref":"T301949#8524210"},[],{"anchor":"8524210"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_245\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_859\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_860\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_247\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-anaxudhz66gjhfv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_861\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_862\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-anaxudhz66gjhfv","anchor":"8524210"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lw6s5scexefzizm\/","ref":"T301949#8527242"},[],{"anchor":"8527242"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_249\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_868\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_869\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_251\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lw6s5scexefzizm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_870\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_871\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lw6s5scexefzizm","anchor":"8527242"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pocj5c5bsotxv6z\/","ref":"T301949#8527645"},[],{"anchor":"8527645"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_253\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_877\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_878\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_255\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pocj5c5bsotxv6z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_879\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_880\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-24mmfocst5ywszi","anchor":"8527645"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mszgqxlmvzoga7t\/","ref":"T301949#8527836"},[],{"anchor":"8527836"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_257\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_886\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_887\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_259\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mszgqxlmvzoga7t\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_888\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_889\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mszgqxlmvzoga7t","anchor":"8527836"}],"javelin_behaviors":{"phui-hovercards":[],"phui-dropdown-menu":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}