for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_58\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6124021\" id=\"6124021\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_57\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eelukey\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6124021\" data-sigil=\"has-tooltip\" data-meta=\"0_56\"\u003e\u003cspan class=\"screen-only\"\u003eMay 11 2020, 10:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-11 10:23:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6128039\" id=\"6128039\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_60\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6128039\" data-sigil=\"has-tooltip\" data-meta=\"0_59\"\u003e\u003cspan class=\"screen-only\"\u003eMay 12 2020, 5:21 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-12 05:21:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_61\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1025\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1025\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003eRadar\u003c\/a\u003e on the \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003eSRE\u003c\/a\u003e board.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6128113\" id=\"6128113\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_70\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6128113\" data-sigil=\"has-tooltip\" data-meta=\"0_69\"\u003e\u003cspan class=\"screen-only\"\u003eMay 12 2020, 6:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-12 06:08:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_67\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_68\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_15\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 595810 had a related patch set uploaded (by Elukey; owner: Elukey):\u003cbr \/\u003e\n[operations\/puppet@production] Remove mc1036\/mc2036 from the Redis Nutcracker config\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/595810\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/595810\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_74\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6128114\" id=\"6128114\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_73\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6128114\" data-sigil=\"has-tooltip\" data-meta=\"0_72\"\u003e\u003cspan class=\"screen-only\"\u003eMay 12 2020, 6:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-12 06:08:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6128118\" id=\"6128118\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_76\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003eelukey\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/user-elukey\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003eUser-Elukey\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6128118\" data-sigil=\"has-tooltip\" data-meta=\"0_75\"\u003e\u003cspan class=\"screen-only\"\u003eMay 12 2020, 6:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-12 06:11:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6132285\" id=\"6132285\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_85\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6132285\" data-sigil=\"has-tooltip\" data-meta=\"0_84\"\u003e\u003cspan class=\"screen-only\"\u003eMay 13 2020, 9:05 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-13 09:05:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_82\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_83\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_21\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSome notes:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdded \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/000000174\/redis?panelId=14&fullscreen&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-job=redis_sessions\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana.wikimedia.org\/d\/000000174\/redis?panelId=14&fullscreen&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-job=redis_sessions\u003c\/a\u003e to the Redis dashboard, that illustrate how evictions are already happening. Max memory is ~500g, and if maxed out the policy that we set is \u003ctt class=\"remarkup-monospaced\"\u003emaxmemory-policy volatile-lru\u003c\/tt\u003e (so we are effectively using Redis as LRU cache, with disk persistence).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMW group2 wikis are not yet using the new session store, so updating nutcracker should be postponed to when all sessions are out of Redis (to minimize user impact).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemoving the nutcracker shards in \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/595810\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/595810\u003c\/a\u003e will re-hash their (mc1036\/2036) keys to the rest of the shards, so not all the keys will be moved\/shifted around.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6132518\" id=\"6132518\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-spinner phui-timeline-icon\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003eelukey\u003c\/a\u003e changed the task status from \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eStalled\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6132518\" data-sigil=\"has-tooltip\" data-meta=\"0_93\"\u003e\u003cspan class=\"screen-only\"\u003eMay 13 2020, 9:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-13 09:56:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_91\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_92\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_23\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ePrecisely, let's hold this task until \u003ca href=\"\/T243106\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT243106\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e is completed.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_104\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6135879\" id=\"6135879\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6135879\" data-sigil=\"has-tooltip\" data-meta=\"0_102\"\u003e\u003cspan class=\"screen-only\"\u003eMay 14 2020, 6:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-14 06:12:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_100\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIn a separate task, I mentioned the following:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eon every mcXXXX we have ~25GB of free RAM (not even used by page cache) that we currently don't use. Even if we'd allocate 10G on every host to be conservative, we'd end up adding 180G in total (we have ~1600G allocated in eqiad, 89G for each shard). That would be ~11% more capacity only using what we currently have. I'd suggest to start with this and see if the cluster overall performance improves (less evictions, higher get hit rate, etc..).\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWhen we'll move to Buster I'd also take the opportunity to use more RAM, it is there and it should be used in my opinion :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6139799\" id=\"6139799\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_106\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eelukey\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/2398\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/2398\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eMcrouter\/Memcached\u003c\/a\u003e on the \u003ca href=\"\/tag\/user-elukey\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eUser-Elukey\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6139799\" data-sigil=\"has-tooltip\" data-meta=\"0_105\"\u003e\u003cspan class=\"screen-only\"\u003eMay 15 2020, 8:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-05-15 08:56:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_116\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6205197\" id=\"6205197\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_115\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6205197\" data-sigil=\"has-tooltip\" data-meta=\"0_114\"\u003e\u003cspan class=\"screen-only\"\u003eJun 9 2020, 9:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-09 09:56:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_112\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_113\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 603942 had a related patch set uploaded (by Elukey; owner: Elukey):\u003cbr \/\u003e\n[operations\/puppet@production] [WIP] memcached: allow more tunables to avoid implicit settings\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/603942\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/603942\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6213725\" id=\"6213725\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_124\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6213725\" data-sigil=\"has-tooltip\" data-meta=\"0_123\"\u003e\u003cspan class=\"screen-only\"\u003eJun 11 2020, 6:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-11 06:00:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_121\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 603942 \u003cstrong\u003emerged\u003c\/strong\u003e by Elukey:\u003cbr \/\u003e\n[operations\/puppet@production] memcached: allow more tunables to avoid implicit settings\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/603942\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/603942\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6223839\" id=\"6223839\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_133\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6223839\" data-sigil=\"has-tooltip\" data-meta=\"0_132\"\u003e\u003cspan class=\"screen-only\"\u003eJun 15 2020, 1:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-15 13:04:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_130\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_131\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_35\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eA little note about the last patch merged. There are two main memcached parameters that can influence the distribution of the slab classes' chunk size: growth factor and smallest chunk size. The algorithm used by memcached 1.5.x is something like the following:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eimport argparse\nimport math\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(description='Generate slab list for memcached.')\n    parser.add_argument('f', type=float, help='Chunk size growth factor \/ -f parameter')\n    parser.add_argument('n', type=int, help='Minimum space allocated for key+value+flags')\n\n    args = parser.parse_args()\n    growth_factor = args.f\n    # sizeof(item) + chunk_size\n    chunk_size = 48 + args.n\n    slab = 1\n    chunk_align = 8\n    max_slab_reached = False\n\n    while True:\n        if chunk_size % chunk_align:\n            chunk_size += chunk_align - (chunk_size % chunk_align)\n        if chunk_size >= 512000:\n            chunk_size = 512000\n            max_slab_reached = True\n        print("Slab: {} Chunk: {}".format(str(slab), str(math.floor(chunk_size))))\n        chunk_size = math.floor(chunk_size * growth_factor)\n        slab += 1\n        if slab >= 64 or max_slab_reached:\n            break\n        if slab == 63:\n            print("Slab: 63 Chunk: 512000")\n            break\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSince years ago, we use growth factor \u003ctt class=\"remarkup-monospaced\"\u003e1.05\u003c\/tt\u003e and smallest chunk size \u003ctt class=\"remarkup-monospaced\"\u003e5 bytes\u003c\/tt\u003e. This is the distribution of chunk sizes for the Gutter Pool for example:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e~ python3 memc_growth_distrib.py 1.15 5\nSlab: 1 Chunk: 56\nSlab: 2 Chunk: 64\nSlab: 3 Chunk: 80\nSlab: 4 Chunk: 96\nSlab: 5 Chunk: 112\nSlab: 6 Chunk: 128\nSlab: 7 Chunk: 152\nSlab: 8 Chunk: 176\nSlab: 9 Chunk: 208\n[..]\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe distribution can easily checked in \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/000000317\/memcache-slabs?panelId=97&fullscreen&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-cluster=memcached_gutter&var-instance=mc-gp1001&var-slab=All\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003egrafana\u003c\/a\u003e. There is one little gotcha though, namely that the biggest slab is capped, by default, to the max chunk size that is 512K (at least this is my understanding from reading docs + code). So running our script above we get:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e[..]\nSlab: 52 Chunk: 94592\nSlab: 53 Chunk: 108784\nSlab: 54 Chunk: 125104\nSlab: 55 Chunk: 143872\nSlab: 56 Chunk: 165456\nSlab: 57 Chunk: 190280\nSlab: 58 Chunk: 218824\nSlab: 59 Chunk: 251648\nSlab: 60 Chunk: 289400\nSlab: 61 Chunk: 332816\nSlab: 62 Chunk: 382744\nSlab: 63 Chunk: 512000\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eBut in reality there is a big jump between 62 and 63, namely 383k -> 512K. Items bigger than 512K and up to 1MB (max size limit for a key, now configurable) will all be stored in the last slab, using one or more chunks "glued" together.\u003c\/p\u003e\n\n\u003cp\u003eThere is also another relevant thing - how come that the smallest chunk size that we set is 5B, but the first slab class is 56B? The reason is that memcached will add additional 48B for the key's metadata+book-keeping, and it will round the size to multiple of 8B for alignment purposes. If we check \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/000000317\/memcache-slabs?panelId=97&fullscreen&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-cluster=memcached&var-instance=mc1027&var-slab=All\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003egrafana\u003c\/a\u003e for mc1027 (or any other shard) we notice that the first slabs are around 80\/90B, so we should think about using the default starting slab size at 48B.\u003c\/p\u003e\n\n\u003cp\u003eThe distribution will become:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e~ python3 memc_growth_distrib.py 1.15 48\nSlab: 1 Chunk: 96\nSlab: 2 Chunk: 112\nSlab: 3 Chunk: 128\nSlab: 4 Chunk: 152\nSlab: 5 Chunk: 176\nSlab: 6 Chunk: 208\nSlab: 7 Chunk: 240\nSlab: 8 Chunk: 280\nSlab: 9 Chunk: 328\nSlab: 10 Chunk: 384\nSlab: 11 Chunk: 448\nSlab: 12 Chunk: 520\nSlab: 13 Chunk: 600\nSlab: 14 Chunk: 696\nSlab: 15 Chunk: 800\nSlab: 16 Chunk: 920\nSlab: 17 Chunk: 1064\nSlab: 18 Chunk: 1224\nSlab: 19 Chunk: 1408\nSlab: 20 Chunk: 1624\nSlab: 21 Chunk: 1872\nSlab: 22 Chunk: 2152\nSlab: 23 Chunk: 2480\nSlab: 24 Chunk: 2856\nSlab: 25 Chunk: 3288\nSlab: 26 Chunk: 3784\nSlab: 27 Chunk: 4352\nSlab: 28 Chunk: 5008\nSlab: 29 Chunk: 5760\nSlab: 30 Chunk: 6624\nSlab: 31 Chunk: 7624\nSlab: 32 Chunk: 8768\nSlab: 33 Chunk: 10088\nSlab: 34 Chunk: 11608\nSlab: 35 Chunk: 13352\nSlab: 36 Chunk: 15360\nSlab: 37 Chunk: 17664\nSlab: 38 Chunk: 20320\nSlab: 39 Chunk: 23368\nSlab: 40 Chunk: 26880\nSlab: 41 Chunk: 30912\nSlab: 42 Chunk: 35552\nSlab: 43 Chunk: 40888\nSlab: 44 Chunk: 47024\nSlab: 45 Chunk: 54080\nSlab: 46 Chunk: 62192\nSlab: 47 Chunk: 71520\nSlab: 48 Chunk: 82248\nSlab: 49 Chunk: 94592\nSlab: 50 Chunk: 108784\nSlab: 51 Chunk: 125104\nSlab: 52 Chunk: 143872\nSlab: 53 Chunk: 165456\nSlab: 54 Chunk: 190280\nSlab: 55 Chunk: 218824\nSlab: 56 Chunk: 251648\nSlab: 57 Chunk: 289400\nSlab: 58 Chunk: 332816\nSlab: 59 Chunk: 382744\nSlab: 60 Chunk: 440160\nSlab: 61 Chunk: 506184\nSlab: 62 Chunk: 512000\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThat seems to be good to test for me. Thoughts? If nobody opposes I'd try to roll it out to the gutter pool first :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6224582\" id=\"6224582\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6224582\" data-sigil=\"has-tooltip\" data-meta=\"0_141\"\u003e\u003cspan class=\"screen-only\"\u003eJun 15 2020, 3:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-15 15:52:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_139\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 605617 had a related patch set uploaded (by Elukey; owner: Elukey):\u003cbr \/\u003e\n[operations\/puppet@production] role::mediawiki::memcached::gutter: change slab distribution\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/605617\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/605617\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_152\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6226998\" id=\"6226998\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_151\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6226998\" data-sigil=\"has-tooltip\" data-meta=\"0_150\"\u003e\u003cspan class=\"screen-only\"\u003eJun 16 2020, 6:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-16 06:12:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_148\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 605617 \u003cstrong\u003emerged\u003c\/strong\u003e by Elukey:\u003cbr \/\u003e\n[operations\/puppet@production] role::mediawiki::memcached::gutter: change slab distribution\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/605617\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/605617\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6227020\" id=\"6227020\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_160\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6227020\" data-sigil=\"has-tooltip\" data-meta=\"0_159\"\u003e\u003cspan class=\"screen-only\"\u003eJun 16 2020, 6:25 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-16 06:25:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_157\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/-yrNu3IBv7KcG9M-jhcQ\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2020-06-16T06:25:10Z] <elukey> roll restart memcached on mc-gp* (gutter pools) to pick up new slab size distribution setting - \u003ca href=\"\/T252391\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT252391\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_170\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6227047\" id=\"6227047\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6227047\" data-sigil=\"has-tooltip\" data-meta=\"0_168\"\u003e\u003cspan class=\"screen-only\"\u003eJun 16 2020, 6:34 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-16 06:34:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_166\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_167\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe next steps for this task should be:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRemove the nutcracker shards in \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/595810\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/595810\u003c\/a\u003e (the change should re-hash their (mc1036\/2036) keys to the rest of the shards, so not all the keys will be moved\/shifted around). Sessions are not in Redis anymore, so it is less risky but nonetheless we may want to wait until Redis is less used. The downside is that this may not happen in months, so we'll not be able to test memcached on Buster in the meantime (with production traffic I mean, tuning slabs etc..).\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003col class=\"remarkup-list\" start=\"2\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eReimage mc1036 to Buster, using the gutter pool's settings (possibly tuning the number of threads).\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6230553\" id=\"6230553\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_178\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6230553\" data-sigil=\"has-tooltip\" data-meta=\"0_177\"\u003e\u003cspan class=\"screen-only\"\u003eJun 17 2020, 6:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-17 06:16:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_175\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_176\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNote after checking slab distribution on the gutter pool. The last slab sizes seem to not follow the prediction made by the script:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSTAT 49:chunk_size 94592\nSTAT 50:chunk_size 108784\nSTAT 51:chunk_size 125104\nSTAT 52:chunk_size 143872\nSTAT 53:chunk_size 165456\nSTAT 54:chunk_size 190280\nSTAT 55:chunk_size 218824\nSTAT 56:chunk_size 251648\nSTAT 57:chunk_size 289400\nSTAT 58:chunk_size 332816\nSTAT 59:chunk_size 382744\nSTAT 61:chunk_size 524288\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eIt is all fine up to slab 59, then 60 is missing (probably due to the absence of anything stored, but should be there at around ~440k) and 61 is at 524k, so there is something that I didn't take into account in the script. Will check, but it should look good anyway, let's see how it behaves.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_188\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6247624\" id=\"6247624\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_187\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eelukey\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eKrinkle\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6247624\" data-sigil=\"has-tooltip\" data-meta=\"0_186\"\u003e\u003cspan class=\"screen-only\"\u003eJun 23 2020, 8:34 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-23 08:34:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_184\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_185\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eGetting back to \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/#\/c\/operations\/puppet\/+\/595810\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/#\/c\/operations\/puppet\/+\/595810\/\u003c\/a\u003e - one thing that it would be useful before merging is to dump all the keys on mc1036 and get a breakdown of the content.\u003c\/p\u003e\n\n\u003cp\u003eStarted with \u003ctt class=\"remarkup-monospaced\"\u003eredis-cli -a "$(sudo grep -Po '(?<=masterauth ).*' \/etc\/redis\/tcp_6379.conf)" -p 6379 KEYS \\* > keys.txt\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eelukey@mc1036:~$ wc -l keys.txt\n379432 keys.txt\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe majority of the keys (~81%) are related to \u003ctt class=\"remarkup-monospaced\"\u003ecentralauth::session\u003c\/tt\u003e, that in theory should have been migrated to cask?\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eelukey@mc1036:~$ grep 'centralauth:session' keys.txt | wc -l\n308898\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThen we have global loginnotify prevSubnet:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eelukey@mc1036:~$ grep global:loginnotify:prevSubnet keys.txt | wc -l\n42740\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThen we have chronology protector:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eelukey@mc1036:~$ grep 'Wikimedia\\\\Rdbms\\\\ChronologyProtector' keys.txt | wc -l\n17593\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd OAUTH tokens:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eelukey@mc1036:~$ grep OAUTH keys.txt | wc -l\n5337\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe remaining keys are:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eelukey@mc1036:~$ cat keys.txt| grep -v 'Wikimedia\\\\Rdbms\\\\ChronologyProtector' | grep -v 'centralauth:session' | grep -v OAUTH | grep -v global:loginnotify:prevSubnet| awk -F ":" '{$(NF--)=""; print}' | sort | uniq -c | sort -n -k 1\n      1 aawiktionary abusefilter-profile group\n      1 angwiki abusefilter-profile group\n      1 arbcom_cswiki abusefilter-profile group\n      1 arwiki abusefilter profile 22\n      1 arwiki abusefilter profile 61\n      1 arwiki abusefilter-profile group\n      1 arwiki abusefilter throttle 106 user\n      1 arwikisource abusefilter-profile v3\n      1 arzwiki captcha\n      1 aywiki abusefilter-profile group\n      1 azwiki captcha\n      1 bdwikimedia abusefilter-profile group\n      1 betawikiversity captcha\n      1 bewiki abusefilter-profile v3\n      1 bswikiquote captcha\n      1 bswikisource captcha\n      1 cawikiquote abusefilter-profile group\n      1 centralauth centralautologin-token \n      1 centralauth centralautologin-token \n      1 chrwiktionary captcha\n      1 commonswiki abusefilter profile 117\n      1 commonswiki abusefilter profile 141\n      1 commonswiki abusefilter profile 154\n      1 commonswiki abusefilter profile 16\n      1 commonswiki abusefilter profile 166\n      1 commonswiki abusefilter profile 168\n      1 commonswiki abusefilter profile 170\n      1 commonswiki abusefilter profile 57\n      1 commonswiki abusefilter profile 89\n      1 commonswiki sitestatsupdate pendingdelta ss_images\n      1 crhwiki captcha\n      1 csbwiki captcha\n      1 cswiki captcha\n      1 cswiki editor-journey\n      1 dewikibooks captcha\n      1 dewikinews captcha\n      1 dewikiquote flaggedrevs\n      1 dewikisource captcha\n      1 dewiktionary flaggedrevs\n      1 dsbwiki captcha\n      1 dvwiki captcha\n      1 eewiki abusefilter-profile group\n      1 elwiki abusefilter-profile group\n      1 elwiki abusefilter-profile v3\n      1 elwikivoyage abusefilter-profile group\n      1 enwiki abusefilter profile 172\n      1 enwiki abusefilter profile 260\n      1 enwiki abusefilter profile 425\n      1 enwiki abusefilter profile 61\n      1 enwiki abusefilter profile 932\n      1 enwiki abusefilter profile 989\n      1 enwikibooks abusefilter-profile v3\n      1 enwikinews abusefilter-profile v3\n      1 enwikinews flaggedrevs\n      1 enwikivoyage abusefilter-profile group\n      1 eswiki abusefilter profile 42\n      1 eswiki abusefilter profile 43\n      1 eswiki abusefilter profile 56\n      1 eswiki abusefilter profile 89\n      1 eswiki abusefilter profile 9\n      1 eswikibooks abusefilter-profile v3\n      1 eswikinews abusefilter profile 18\n      1 eswikinews captcha\n      1 eswikinews flaggedrevs\n      1 eswikiquote abusefilter-profile group\n      1 eswikiquote captcha\n      1 eswikivoyage abusefilter-profile v3\n      1 euwiki abusefilter profile 169\n      1 euwikibooks abusefilter-profile group\n      1 euwiki editor-journey\n      1 fawiki abusefilter profile 125\n      1 fawiki abusefilter-profile group\n      1 fawikiquote abusefilter-profile group\n      1 fawikiquote abusefilter-profile v3\n      1 fawikivoyage captcha\n      1 fiwiki abusefilter profile 6\n      1 frwiki abusefilter profile 19\n      1 frwiki abusefilter profile 214\n      1 frwiki abusefilter profile 242\n      1 frwiki abusefilter profile 278\n      1 frwikinews flaggedrevs\n      1 frwikiquote abusefilter-profile v3\n      1 frwikisource abusefilter-profile group\n      1 fywikibooks abusefilter-profile group\n      1 ganwiki abusefilter-profile group\n      1 gdwiki captcha\n      1 global abusefilter throttle metawiki 206 user\n      1 global watchlist-recent-updates arwiki\n      1 global watchlist-recent-updates arzwiki\n      1 global watchlist-recent-updates azwiki\n      1 global watchlist-recent-updates be_x_oldwiki\n      1 global watchlist-recent-updates bgwiki\n      1 global watchlist-recent-updates cawiki\n      1 global watchlist-recent-updates cswiki\n      1 global watchlist-recent-updates dewikivoyage\n      1 global watchlist-recent-updates enwiktionary\n      1 global watchlist-recent-updates fawiki\n      1 global watchlist-recent-updates fawikivoyage\n      1 global watchlist-recent-updates frwiktionary\n      1 global watchlist-recent-updates hiwiki\n      1 global watchlist-recent-updates itwikiquote\n      1 global watchlist-recent-updates rmwiki\n      1 global watchlist-recent-updates tewiki\n      1 global watchlist-recent-updates thwiki\n      1 global watchlist-recent-updates trwiki\n      1 glwiki abusefilter profile 15\n      1 glwiki abusefilter-profile v3\n      1 glwikiquote abusefilter-profile group\n      1 gnwiki captcha\n      1 gomwiki abusefilter-profile group\n      1 guwikisource abusefilter-profile group\n      1 gvwiktionary captcha\n      1 hawiki captcha\n      1 hewiki abusefilter profile 47\n      1 hewiki abusefilter profile 53\n      1 hewikisource abusefilter profile 132\n      1 hiwiki abusefilter-profile v3\n      1 hrwikibooks abusefilter-profile group\n      1 hrwiki captcha\n      1 huwikibooks captcha\n      1 huwiki captcha\n      1 hywiki abusefilter profile 196\n      1 hywiki abusefilter-profile v3\n      1 hywiki captcha\n      1 hywiki newcomer-tasks\n      1 idwiki abusefilter profile 8\n      1 idwiki abusefilter-profile v3\n      1 idwikibooks abusefilter-profile v3\n      1 idwikiquote captcha\n      1 incubatorwiki abusefilter profile 176\n      1 incubatorwiki captcha\n      1 itwiki abusefilter profile 267\n      1 itwiki abusefilter profile 353\n      1 itwiki abusefilter profile 394\n      1 itwiki abusefilter profile 423\n      1 itwiki abusefilter profile 472\n      1 itwikinews abusefilter-profile v3\n      1 itwikinews captcha\n      1 itwikiversity abusefilter-profile v3\n      1 itwikiversity captcha\n      1 itwikivoyage abusefilter-profile v3\n      1 itwiktionary abusefilter-profile v3\n      1 iuwiktionary captcha\n      1 jawikiquote abusefilter-profile v3\n      1 jawikisource abusefilter-profile v3\n      1 jvwiki captcha\n      1 kabwiki abusefilter-profile group\n      1 kbpwiki abusefilter-profile group\n      1 kmwikibooks captcha\n      1 knwiki abusefilter-profile v3\n      1 kowiktionary abusefilter-profile v3\n      1 kywiki abusefilter-profile group\n      1 lfnwiki captcha\n      1 lnwiktionary captcha\n      1 loginwiki abusefilter profile 95\n      1 ltgwiki captcha\n      1 ltwiki captcha\n      1 mediawikiwiki sitestatsupdate pendingdelta ss_total_pages\n      1 metawiki abusefilter profile 101\n      1 metawiki abusefilter profile 181\n      1 metawiki abusefilter-profile group\n      1 metawiki translate-translator-activity-v1\n      1 miwiki captcha\n      1 mnwwiki captcha\n      1 mswiki abusefilter-profile v3\n      1 mswikibooks abusefilter-profile group\n      1 mswiki captcha\n      1 mwlwiki abusefilter-profile group\n      1 mywiki abusefilter-profile v3\n      1 mznwiki abusefilter-profile group\n      1 nawiki captcha\n      1 nawiktionary captcha\n      1 nlwiki abusefilter-profile v3\n      1 nlwikibooks captcha\n      1 nlwikimedia captcha\n      1 nlwikinews captcha\n      1 nlwiktionary captcha\n      1 ocwiki abusefilter-profile group\n      1 olowiki captcha\n      1 orwiki captcha\n      1 orwikisource captcha\n      1 orwiktionary captcha\n      1 pamwiki abusefilter-profile group\n      1 papwiki captcha\n      1 piwiki captcha\n      1 plwiki abusefilter-profile group\n      1 plwikisource abusefilter profile 2\n      1 plwiktionary abusefilter-profile v3\n      1 ptwiki abusefilter profile 114\n      1 ptwiki abusefilter profile 120\n      1 ptwiki abusefilter profile 139\n      1 ptwiki abusefilter profile 172\n      1 ptwiki abusefilter profile 31\n      1 ptwiki abusefilter profile 94\n      1 ptwikibooks captcha\n      1 ptwikinews abusefilter profile 123\n      1 ptwikinews abusefilter profile 72\n      1 ptwikiquote captcha\n      1 ptwikisource abusefilter-profile v3\n      1 ptwikisource captcha\n      1 ptwiktionary abusefilter-profile v3\n      1 ptwiktionary captcha\n      1 rnwiki abusefilter-profile group\n      1 rowiki abusefilter profile 53\n      1 rowiki captcha\n      1 ruewiki captcha\n      1 ruwiki abusefilter throttle 17 ip\n      1 ruwikinews captcha\n      1 ruwikiquote captcha\n      1 rwwiki abusefilter-profile group\n      1 sawiki abusefilter-profile group\n      1 sawiki abusefilter-profile v3\n      1 sawikisource captcha\n      1 scnwiki captcha\n      1 sdwiki abusefilter-profile v3\n      1 siwiki captcha\n      1 skwiki abusefilter throttle 33 user,page\n      1 specieswiki abusefilter-profile v3\n      1 suwiki captcha\n      1 svwiki abusefilter profile 33\n      1 svwiki abusefilter profile 53\n      1 svwiki abusefilter profile 89\n      1 svwiktionary abusefilter-profile v3\n      1 szlwiki abusefilter-profile group\n      1 tawiki abusefilter profile 181\n      1 tawiki abusefilter-profile v3\n      1 tawiki captcha\n      1 tawikisource captcha\n      1 tawiktionary captcha\n      1 tcywiki captcha\n      1 testwiki centralnotice bannerfields\n      1 tewiki captcha\n      1 tkwiki abusefilter-profile group\n      1 tpiwiktionary abusefilter-profile group\n      1 trwikisource abusefilter-profile group\n      1 trwiktionary abusefilter-profile group\n      1 trwiktionary abusefilter-profile v3\n      1 trwiktionary captcha\n      1 tumwiki captcha\n      1 ukwiki abusefilter-profile v3\n      1 ukwiki captcha\n      1 urwiki abusefilter profile 169\n      1 urwiki captcha\n      1 viwiki abusefilter-profile v3\n      1 viwiki newcomer-tasks\n      1 warwiki abusefilter-profile group\n      1 wikidatawiki abusefilter profile 108\n      1 wikidatawiki abusefilter profile 111\n      1 wikidatawiki abusefilter profile 87\n      1 wikidatawiki abusefilter profile 92\n      1 wikidatawiki abusefilter throttle new user,page\n      1 yowiki captcha\n      1 zh_min_nanwikisource captcha\n      1 zhwiki abusefilter profile 127\n      1 zhwiki abusefilter profile 231\n      1 zhwiki abusefilter profile 69\n      1 zhwiki abusefilter throttle 253 user,ip\n      1 zhwikivoyage abusefilter-profile v3\n      1 zh_yuewiki abusefilter-profile v3\n      2 atjwiki captcha\n      2 azwiki abusefilter-profile v3\n      2 bnwiki abusefilter-profile v3\n      2 cawiki abusefilter-profile v3\n      2 cswiktionary abusefilter-profile v3\n      2 dewikibooks abusefilter-profile v3\n      2 dewiktionary abusefilter-profile v3\n      2 dewiktionary captcha\n      2 eewiki captcha\n      2 elwiki captcha\n      2 enwiki abusefilter throttle 806 user\n      2 enwikiquote abusefilter-profile v3\n      2 enwikiquote captcha\n      2 enwikisource abusefilter-profile v3\n      2 frwiktionary abusefilter-profile v3\n      2 global watchlist-recent-updates elwiki\n      2 global watchlist-recent-updates glwiki\n      2 global watchlist-recent-updates kowiki\n      2 global watchlist-recent-updates svwiki\n      2 hiwiki captcha\n      2 huwiki editor-journey\n      2 iewiki captcha\n      2 incubatorwiki abusefilter-profile v3\n      2 jawiki abusefilter-profile v3\n      2 jawikisource captcha\n      2 kkwiki abusefilter-profile v3\n      2 ltwiki abusefilter-profile v3\n      2 lvwiki captcha\n      2 maiwiki abusefilter-profile v3\n      2 mediawikiwiki abusefilter-profile v3\n      2 metawiki captcha\n      2 mrwiki abusefilter throttle 127 page\n      2 mywiki captcha\n      2 ruwikiquote abusefilter-profile v3\n      2 ruwiktionary abusefilter-profile v3\n      2 skwiki abusefilter-profile v3\n      2 slwiki captcha\n      2 sourceswiki captcha\n      2 thwiki captcha\n      2 uzwiki captcha\n      2 vecwiki abusefilter-profile v3\n      2 zhwikinews abusefilter-profile v3\n      3 bgwiki abusefilter-profile v3\n      3 bjnwiki captcha\n      3 centralauth centralautologin-token\n      3 dewiki abusefilter throttle 242 user\n      3 etwiki captcha\n      3 frwikiversity captcha\n      3 global watchlist-recent-updates idwiki\n      3 global watchlist-recent-updates metawiki\n      3 global watchlist-recent-updates rowiki\n      3 guwiki abusefilter-profile v3\n      3 newiki abusefilter-profile v3\n      3 plwiki abusefilter-profile v3\n      3 specieswiki captcha\n      3 trwiki abusefilter-profile v3\n      3 ukwiki editor-journey\n      4 centralauth api-token\n      4 ckbwiki abusefilter-profile v3\n      4 cswiki newcomer-tasks\n      4 enwikivoyage abusefilter-profile v3\n      4 eswiki abusefilter-profile v3\n      4 global watchlist-recent-updates huwiki\n      4 global watchlist-recent-updates nlwiki\n      4 global watchlist-recent-updates ptwiki\n      4 kowiki newcomer-tasks\n      4 nlwiki captcha\n      4 testwiki abusefilter-profile v3\n      4 wikidatawiki abusefilter-profile v3\n      4 wikidatawiki captcha\n      5 cswiki abusefilter-profile v3\n      5 dewiki abusefilter-profile v3\n      5 enwiki abusefilter throttle 279 user,page\n      5 enwiktionary abusefilter-profile v3\n      5 euwiki captcha\n      5 fawiki captcha\n      5 frwiktionary captcha\n      5 global watchlist-recent-updates plwiki\n      5 global watchlist-recent-updates ukwiki\n      5 global watchlist-recent-updates zhwiki\n      5 huwiki abusefilter-profile v3\n      5 kowiki editor-journey\n      5 shwiki captcha\n      5 svwiki abusefilter-profile v3\n      6 commonswiki abusefilter-profile v3\n      6 fiwiki abusefilter-profile v3\n      6 fiwiki captcha\n      6 global watchlist-recent-updates ruwiki\n      6 global watchlist-recent-updates wikidatawiki\n      6 hewiki abusefilter-profile v3\n      6 metawiki abusefilter-profile v3\n      6 rowiki abusefilter-profile v3\n      6 ruwiki abusefilter-profile v3\n      6 viwiki captcha\n      7 ptwiki abusefilter-profile v3\n      7 simplewiki abusefilter-profile v3\n      7 viwiki editor-journey\n      7 zhwiki abusefilter-profile v3\n      8 arwiki abusefilter-profile v3\n      8 arwiki captcha\n      8 enwiki abusefilter-profile v3\n      8 enwikivoyage captcha\n      8 global watchlist-recent-updates eswiki\n      8 global watchlist-recent-updates hewiki\n      8 mrwiki abusefilter-profile v3\n      8 trwiki captcha\n      9 frwiki newcomer-tasks\n      9 thwiki abusefilter-profile v3\n     10 fawiki abusefilter-profile v3\n     10 frwiki abusefilter-profile v3\n     10 itwiki abusefilter-profile v3\n     10 simplewiktionary captcha\n     11 arwiki newcomer-tasks\n     11 global watchlist-recent-updates jawiki\n     11 itwiki captcha\n     11 jawiki captcha\n     12 global watchlist-recent-updates commonswiki\n     13 ruwiki captcha\n     14 global watchlist-recent-updates itwiki\n     15 arwiki abusefilter throttle 175 user\n     16 global watchlist-recent-updates frwiki\n     16 zhwiki captcha\n     17 mediawikiwiki captcha\n     20 plwiki captcha\n     21 enwikiversity captcha\n     21 enwiktionary captcha\n     21 idwiki captcha\n     23 arwiki editor-journey\n     30 simplewiki captcha\n     33 global watchlist-recent-updates dewiki\n     34 arwiki abusefilter throttle 175 ip\n     37 eswiki captcha\n     60 ptwiki captcha\n     65 mrwiki abusefilter throttle 9 user,ip\n     75 testwiki ResourceLoaderModule-dependencies\n     89 cswiki abusefilter throttle new ip,page\n     89 global watchlist-recent-updates enwiki\n    101 cswiki abusefilter throttle new user,page\n    111 global loginnotify known\n    180 global loginnotify new\n    204 frwiki captcha\n    277 dewiki captcha\n    449 mediawikiwiki ResourceLoaderModule-dependencies\n    585 commonswiki captcha\n    680 enwiki captcha\n    792 metawiki centralnotice bannerfields\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Krinkle\u003c\/span\u003e\u003c\/a\u003e do you know what kind of impact users can have if the above keys would disappear? (to be recreated later on in other shards).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_199\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kriyar3k3eptnbjnvd4q\/PHID-FILE-enywe2vuj2f26zdgviqk\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/RLazarus\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/22\/\" data-sigil=\"has-tooltip\" data-meta=\"0_197\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-graduation-cap\" data-meta=\"0_198\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6248986\" id=\"6248986\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_196\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RLazarus\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003eRLazarus\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6248986\" data-sigil=\"has-tooltip\" data-meta=\"0_195\"\u003e\u003cspan class=\"screen-only\"\u003eJun 23 2020, 4:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-23 16:33:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_193\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_194\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSide note: This question is also interesting from a DC switchover perspective (\u003ca href=\"\/T243316\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT243316\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e) since that will also effectively be a Redis flush. In previous switchovers we only explicitly handled replication for sessions data, and now that's out of Redis. If there's anything else in there that we can't afford to drop and recreate, now would be a great time to know that.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_208\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6296113\" id=\"6296113\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_207\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eelukey\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003eaaron\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6296113\" data-sigil=\"has-tooltip\" data-meta=\"0_206\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2020, 8:47 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-07-10 08:47:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_204\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_205\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Krinkle\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/p\/aaron\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aaron\u003c\/span\u003e\u003c\/a\u003e if you have time, let's follow up on the question that I asked about what happens if a Redis shard disappears. It would be really nice to start testing a new version of memcached this quarter :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_217\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6296409\" id=\"6296409\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6296409\" data-sigil=\"has-tooltip\" data-meta=\"0_215\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2020, 10:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-07-10 10:26:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_213\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_214\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnother idea to add in here - recently John and Moritz needed TLS for memcached and imported memcached 1.6.6 (latest upstream) into out buster repositories. We could think about moving directly to 1.6.6 for some reasons:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eNative TLS support\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eexstore \u003ca href=\"https:\/\/memcached.org\/blog\/nvm-caching\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/memcached.org\/blog\/nvm-caching\/\u003c\/a\u003e\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003ePoint 2) is very interesting for two long term reasons:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf possible, adding NVMe to the gutter pool could be an easy way to sustain more load than 256G of ram (assuming no saturation of the 10G NIC of course)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhen we'll need to refresh mc*, it might be possible to think about shrinking the current 18 shards into something smaller, leveraging extstore and NVMe.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eNot saying that we have to do it, but just to think about it :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-p6eirdqromiu22okspzs"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-rh5pc4rp6zawd5z3lixz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-p6eirdqromiu22okspzs"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz","contextPHID":"PHID-TASK-p6eirdqromiu22okspzs"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jx6ahure6z4vkiwcszyu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz","contextPHID":"PHID-TASK-p6eirdqromiu22okspzs"}},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq","contextPHID":"PHID-TASK-p6eirdqromiu22okspzs"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-c4yrc5wlq7aalby54odc"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-zy6a5iabqzrixazhme2s"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-5is7l5fh7zc7hlr"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-zasg3muv4ljulhoy73kv"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-k3skulu74eeoqfq"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-hmtsieubzkmpwed"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-m7je4td3llql6bv"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-nf7uo6m7vlgijvqrkcdq"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ymyalm5jne6yb7kkpo7s"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-zasg3muv4ljulhoy73kv"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-6uhf74aqktthkuv"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-cc32ksqs62bygbm"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-tznmlo5ktphdief"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-4jh7hgmmykljpvm"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-r52vnuyggjy2els"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-ovzh2zaa5uj3gsi"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-qfz3qcbatahfrzm"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-jzb7nu5eygv3u2m"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"phid":"PHID-XACT-TASK-l2v7yryjjiqxkbw"},{"hovercardSpec":{"objectPHID":"PHID-USER-gvrjiwuwofby2t5ofb2e"}},{"phid":"PHID-XACT-TASK-oxn626rzo25x77q"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"phid":"PHID-XACT-TASK-xeztqzxqrne75ww"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-abntebbkcoqwplm"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-psq2xjeddbjyqyj","anchor":"6124021"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-dwvh2mfwatoosmm","anchor":"6128039"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-5is7l5fh7zc7hlr\/","ref":"T252391#6128113"},[],{"anchor":"6128113"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_63\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_64\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5is7l5fh7zc7hlr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_65\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_66\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5is7l5fh7zc7hlr","anchor":"6128113"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dhwfxaap66rvkfu","anchor":"6128114"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ho2clwupq2p6ktq","anchor":"6128118"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-k3skulu74eeoqfq\/","ref":"T252391#6132285"},[],{"anchor":"6132285"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_78\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-k3skulu74eeoqfq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_80\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-k3skulu74eeoqfq","anchor":"6132285"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-hmtsieubzkmpwed\/","ref":"T252391#6132518"},[],{"anchor":"6132518"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_87\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hmtsieubzkmpwed\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_89\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-q6hceanqho5qghw","anchor":"6132518"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-m7je4td3llql6bv\/","ref":"T252391#6135879"},[],{"anchor":"6135879"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_96\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_97\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-m7je4td3llql6bv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_98\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_99\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-m7je4td3llql6bv","anchor":"6135879"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ja7bjc5dlxrwr43","anchor":"6139799"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-6uhf74aqktthkuv\/","ref":"T252391#6205197"},[],{"anchor":"6205197"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_108\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6uhf74aqktthkuv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_110\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6uhf74aqktthkuv","anchor":"6205197"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-cc32ksqs62bygbm\/","ref":"T252391#6213725"},[],{"anchor":"6213725"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cc32ksqs62bygbm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_119\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cc32ksqs62bygbm","anchor":"6213725"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-tznmlo5ktphdief\/","ref":"T252391#6223839"},[],{"anchor":"6223839"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_126\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tznmlo5ktphdief\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_128\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tznmlo5ktphdief","anchor":"6223839"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-4jh7hgmmykljpvm\/","ref":"T252391#6224582"},[],{"anchor":"6224582"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_135\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4jh7hgmmykljpvm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4jh7hgmmykljpvm","anchor":"6224582"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-r52vnuyggjy2els\/","ref":"T252391#6226998"},[],{"anchor":"6226998"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r52vnuyggjy2els\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_146\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-r52vnuyggjy2els","anchor":"6226998"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ovzh2zaa5uj3gsi\/","ref":"T252391#6227020"},[],{"anchor":"6227020"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_153\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_154\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ovzh2zaa5uj3gsi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_155\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ovzh2zaa5uj3gsi","anchor":"6227020"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-qfz3qcbatahfrzm\/","ref":"T252391#6227047"},[],{"anchor":"6227047"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_162\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_163\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qfz3qcbatahfrzm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_164\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_165\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qfz3qcbatahfrzm","anchor":"6227047"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-jzb7nu5eygv3u2m\/","ref":"T252391#6230553"},[],{"anchor":"6230553"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_171\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_172\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jzb7nu5eygv3u2m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_173\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_174\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jzb7nu5eygv3u2m","anchor":"6230553"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-l2v7yryjjiqxkbw\/","ref":"T252391#6247624"},[],{"anchor":"6247624"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_180\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_181\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l2v7yryjjiqxkbw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_182\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_183\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ij4np4rttug7nu4","anchor":"6247624"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-oxn626rzo25x77q\/","ref":"T252391#6248986"},[],{"anchor":"6248986"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_189\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_190\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-oxn626rzo25x77q\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_191\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_192\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Deployment Training Graduate","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-oxn626rzo25x77q","anchor":"6248986"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-xeztqzxqrne75ww\/","ref":"T252391#6296113"},[],{"anchor":"6296113"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_200\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xeztqzxqrne75ww\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_202\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cxbhbvibbqwhfaj","anchor":"6296113"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-abntebbkcoqwplm\/","ref":"T252391#6296409"},[],{"anchor":"6296409"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_209\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-abntebbkcoqwplm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_211\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-abntebbkcoqwplm","anchor":"6296409"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}