for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_136\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4827944\" id=\"4827944\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_135\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eJoe\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4827944\" data-sigil=\"has-tooltip\" data-meta=\"0_134\"\u003e\u003cspan class=\"screen-only\"\u003eDec 17 2018, 2:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-17 14:55:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4827955\" id=\"4827955\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_28\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/2903335\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4827955\" data-sigil=\"has-tooltip\" data-meta=\"0_137\"\u003e\u003cspan class=\"screen-only\"\u003eDec 17 2018, 2:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-17 14:55:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_142\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4830437\" id=\"4830437\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_141\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eJoe\u003c\/a\u003e renamed this task from \u003cspan class=\"phui-timeline-value\"\u003eUse a multi-dc aware store for `wgMainStash` if needed.\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eUse a multi-dc aware store for ObjectCache's MainStash if needed.\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4830437\" data-sigil=\"has-tooltip\" data-meta=\"0_140\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 8:32 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 08:32:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_145\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4831752\" id=\"4831752\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_144\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eEvanProdromou\u003c\/a\u003e claimed this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4831752\" data-sigil=\"has-tooltip\" data-meta=\"0_143\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 4:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 16:29:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/v5wmnfiaelrd6nztxd3a\/PHID-FILE-s57znldbv3qchtewuryd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/daniel\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_154\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4831754\" id=\"4831754\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_153\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/daniel\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003edaniel\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4831754\" data-sigil=\"has-tooltip\" data-meta=\"0_152\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 4:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 16:30:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_150\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_151\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_32\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCalls to getMainObjectStash: \u003ca href=\"https:\/\/codesearch.wmflabs.org\/search\/?q=getMainObjectStash&i=nope&files=&repos=\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/codesearch.wmflabs.org\/search\/?q=getMainObjectStash&i=nope&files=&repos=\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_159\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rus3onc75davarls6nbc\/PHID-FILE-d4tltleyv665iw455lvp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Imarlier\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4832235\" id=\"4832235\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Imarlier\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Imarlier\u003c\/a\u003e edited projects, added \u003ca href=\"\/project\/view\/2923\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003ePerformance-Team (Radar)\u003c\/a\u003e; removed \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003ePerformance-Team\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4832235\" data-sigil=\"has-tooltip\" data-meta=\"0_157\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 7:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 19:17:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kd4xv6kmiqlmbg5i3ynf\/PHID-FILE-q756a47n7hkxmwqviotl\/profile-profile.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aaron\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4832383\" id=\"4832383\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eaaron\u003c\/a\u003e added a parent task: \u003ca href=\"\/T88445\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eT88445: MediaWiki active\/active datacenter investigation and work (tracking)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4832383\" data-sigil=\"has-tooltip\" data-meta=\"0_160\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 8:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 20:19:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kd4xv6kmiqlmbg5i3ynf\/PHID-FILE-q756a47n7hkxmwqviotl\/profile-profile.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aaron\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4832391\" id=\"4832391\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_170\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eaaron\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4832391\" data-sigil=\"has-tooltip\" data-meta=\"0_169\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 8:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 20:22:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_167\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe need persistence and replication. The plan is to use the same store as session for the rest of the object stash usage (probably Cassandra). Flags like WRITE_SYNC might be used in a few callers, and should use appropriate backend requests (e.g. QUOROM_* settings in Cassandra). The callers of the main object stash all need persistence and replication though (callers have already been migrated to stash vs WAN cache and such).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_182\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_180\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_181\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4832455\" id=\"4832455\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4832455\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 8:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 20:38:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_176\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_42\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4832390\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4832390\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/aaron\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aaron\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eWe need persistence and replication. The plan is to use the same store as session for the rest of the object stash usage (probably Cassandra). Flags like WRITE_SYNC might be used in a few callers, and should use appropriate backend requests (e.g. QUOROM_* settings in Cassandra). The callers of the main object stash all need persistence and replication though (callers have already been migrated to stash vs WAN cache and such).\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThe fact we need persistence is quite surprising, given redis does LRU purging right now, has no backups, no disaster recovery system. Even worse, it's served through nutcracker that doesn't even ensure consistency.\u003c\/p\u003e\n\n\u003cp\u003eSo using cassandra for MainStash objects will mean a \u003cstrong\u003esevere\u003c\/strong\u003e performance penalty and I'm not convinced it's justifiable - either in terms of costs or latencies.\u003c\/p\u003e\n\n\u003cp\u003eMoreover, it seems to me that the usage of this datastore is unbound and uncontrolled, while if it is intended to be persisted long-term it should be treated like data on the databases, and we should discuss any new use before it's permitted in production.\u003c\/p\u003e\n\n\u003cp\u003eSo even if we decide to move this to a storage service similar to the sessions one, it will need multitenancy and client whitelisting per-usage, rather than being a random storage for unstructured data like it seems to be now.\u003c\/p\u003e\n\n\u003cp\u003eThis needs a thorough discussion ASAP.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_191\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4832565\" id=\"4832565\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_190\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eEevans\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4832565\" data-sigil=\"has-tooltip\" data-meta=\"0_189\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 9:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 21:07:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_187\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_188\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4832455\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4832455\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e[ ... ]\u003cbr \/\u003e\nThis needs a thorough discussion ASAP.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI was under the impression \u003ca href=\"https:\/\/www.mediawiki.org\/wiki\/Requests_for_comment\/SessionStorageAPI\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ewe had\u003c\/a\u003e.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_200\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4832587\" id=\"4832587\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_199\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4832587\" data-sigil=\"has-tooltip\" data-meta=\"0_198\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2018, 9:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-18 21:10:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_196\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWell that discussion was limited to Session storage, and I stand by the idea that service, and its datastore, shouldn't be concerned with anything else than sessions, unless we come to a further agreement.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_211\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kd4xv6kmiqlmbg5i3ynf\/PHID-FILE-q756a47n7hkxmwqviotl\/profile-profile.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aaron\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4833043\" id=\"4833043\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eaaron\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4833043\" data-sigil=\"has-tooltip\" data-meta=\"0_209\"\u003e\u003cspan class=\"screen-only\"\u003eDec 19 2018, 12:06 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-19 00:06:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_207\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_208\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe current callers don't assume the level of durability as with mysql, just that the data will likely not be randomly removed (e.g. high eviction rate, power outage, network blips). The WAN cache callers can handle a fair amount of that on the other hand.\u003c\/p\u003e\n\n\u003cp\u003eI think the criteria is a bit fuzzy, and I mentioned on another task that modding mcrouter to do replication for a separate logical pool (same servers) would probably be good enough (for now) for our current object stash callers (aside from sessions, which use $wgSessionCacheType with ObjectCache::getInstance and not getMainObjectStash(), though they happen to have the same backing store on wmf). In the interest of moving things forward, that could be a OK compromise, even if we loose disk persistent (e.g. redis rdb files).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_224\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/lvt5pyizsk3uckl4msuc\/PHID-FILE-eiyvx2uovr7zblyb6rkl\/profile-Kosiaris%2C_Alexandros_Sept_2013.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/akosiaris\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_220\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_221\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_222\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4833375\" id=\"4833375\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_219\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/akosiaris\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003eakosiaris\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4833375\" data-sigil=\"has-tooltip\" data-meta=\"0_218\"\u003e\u003cspan class=\"screen-only\"\u003eDec 19 2018, 6:43 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-19 06:43:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_216\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_217\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4833043\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4833043\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/aaron\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aaron\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThe current callers don't assume the level of durability as with mysql, just that the data will likely not be randomly removed (e.g. high eviction rate, power outage, network blips).\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eBut that actually happens. Data IS randomly removed. Looking at the graphs at \u003ca href=\"https:\/\/redis.io\/topics\/lru-cache\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/redis.io\/topics\/lru-cache\u003c\/a\u003e, we can see that in the case of 2.8 (in which we have been for many years), randomness in key removal was expected for a big percentage of the data. So that assumption on the level of the callers is wrong (but it doesn't seem to hurt).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_235\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_233\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4833420\" id=\"4833420\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_232\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4833420\" data-sigil=\"has-tooltip\" data-meta=\"0_231\"\u003e\u003cspan class=\"screen-only\"\u003eDec 19 2018, 7:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-19 07:17:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_229\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_230\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_52\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eLooking at live data, we have at least one shard that's doing evictions (150k of them) and all shards have 10M+ expired keys.\u003c\/p\u003e\n\n\u003cp\u003eIs this behaviour expected from a storage backend for MainStash? Do we assume all data there comes with a TTL?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_244\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2nco5z2tyvhf6dfxheoj\/PHID-FILE-pacxilbspughn725akdz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mark\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4834330\" id=\"4834330\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mark\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003emark\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4834330\" data-sigil=\"has-tooltip\" data-meta=\"0_242\"\u003e\u003cspan class=\"screen-only\"\u003eDec 19 2018, 3:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-19 15:05:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_240\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI am getting the impression here that some things are being rushed and finalized without time for a proper discussion between people\/teams about the different possible solutions and their impact, after this new discovery. Is that because goals are due to be posted now?\u003c\/p\u003e\n\n\u003cp\u003eI think we might have to take a step back and allow for this discussion to run its course and reach consensus, and do what we need to do to not have goal deadlines interfere in the mean time...\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_247\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gjkyraeh6ascv7vkj3ke\/PHID-FILE-czquixv3lk45jved74s2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/CDanis\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4834491\" id=\"4834491\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/CDanis\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003eCDanis\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4834491\" data-sigil=\"has-tooltip\" data-meta=\"0_245\"\u003e\u003cspan class=\"screen-only\"\u003eDec 19 2018, 3:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-19 15:44:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_256\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4835375\" id=\"4835375\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4835375\" data-sigil=\"has-tooltip\" data-meta=\"0_254\"\u003e\u003cspan class=\"screen-only\"\u003eDec 19 2018, 8:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-19 20:07:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_252\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4834329\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4834329\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/mark\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@mark\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI am getting the impression here that some things are being rushed and finalized without time for a proper discussion between people\/teams about the different possible solutions and their impact, after this new discovery. Is that because goals are due to be posted now?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI'm not sure who this is directed at (read: who's goals), but insofar as session storage goes, I don't personally feel any pressure with respect to goals (and I'm assuming we're on track with them either way).  That said, I don't feel at all certain (yet) that I have the full picture, so maybe it does threaten a goal...\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI think we might have to take a step back and allow for this discussion to run its course and reach consensus, and do what we need to do to not have goal deadlines interfere in the mean time...\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgreed, and we should also make some time for post-mortem.  What is most concerning to me is that we could get this deeply into a project (session storage), only to be surprised that most of what we've been storing this whole time aren't sessions at all.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_259\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4870031\" id=\"4870031\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_258\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eEevans\u003c\/a\u003e added a parent task: \u003ca href=\"\/T206016\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003eT206016: Create a service for session storage\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4870031\" data-sigil=\"has-tooltip\" data-meta=\"0_257\"\u003e\u003cspan class=\"screen-only\"\u003eJan 10 2019, 3:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-10 15:49:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_263\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4881160\" id=\"4881160\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_261\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003eEvanProdromou\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/3654\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003eInbox\u003c\/a\u003e to \u003ca href=\"\/project\/board\/3654\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003eNeeds Cleaning - Security, stability, performance, and scalability (TEC1)\u003c\/a\u003e on the \u003ca href=\"\/tag\/platform_engineering\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003ePlatform Engineering\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4881160\" data-sigil=\"has-tooltip\" data-meta=\"0_260\"\u003e\u003cspan class=\"screen-only\"\u003eJan 15 2019, 3:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-15 15:09:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eEvanProdromou\u003c\/a\u003e edited projects, added \u003ca href=\"\/project\/view\/3666\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003ePlatform Engineering (Needs Cleaning - Security, stability, performance, and scalability (TEC1))\u003c\/a\u003e; removed \u003ca href=\"\/tag\/platform_engineering\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003ePlatform Engineering\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_266\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4881168\" id=\"4881168\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003eEvanProdromou\u003c\/a\u003e added a project: \u003ca href=\"\/project\/view\/3697\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003ePlatform Team Workboards (Doing)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4881168\" data-sigil=\"has-tooltip\" data-meta=\"0_264\"\u003e\u003cspan class=\"screen-only\"\u003eJan 15 2019, 3:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-15 15:13:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_275\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4885223\" id=\"4885223\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eEvanProdromou\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4885223\" data-sigil=\"has-tooltip\" data-meta=\"0_273\"\u003e\u003cspan class=\"screen-only\"\u003eJan 16 2019, 5:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-16 17:30:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_271\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_272\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_70\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSo, we use caching in MediaWiki for a ton of different things: parser cache, revision cache, counters, rate limiting, and so on.\u003c\/p\u003e\n\n\u003cp\u003eBy default since 1.27, sessions are stored in the same object cache as anything else, but we can specifically cut out sessions to their own storage class with a configuration variable $wgSessionCacheType.\u003c\/p\u003e\n\n\u003cp\u003eIs a reasonable path forward to:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMove sessions only to the new object store \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e and team are working on, while the rest of the cached objects stay on the current Redis infrastructure\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMake a decision whether and how to move other object types to the same or different object store later?\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_286\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_284\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4885245\" id=\"4885245\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4885245\" data-sigil=\"has-tooltip\" data-meta=\"0_282\"\u003e\u003cspan class=\"screen-only\"\u003eJan 16 2019, 5:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-16 17:37:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_280\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_72\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4885223\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4885223\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@EvanProdromou\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eSo, we use caching in MediaWiki for a ton of different things: parser cache, revision cache, counters, rate limiting, and so on.\u003c\/p\u003e\n\n\u003cp\u003eBy default since 1.27, sessions are stored in the same object cache as anything else, but we can specifically cut out sessions to their own storage class with a configuration variable $wgSessionCacheType.\u003c\/p\u003e\n\n\u003cp\u003eIs a reasonable path forward to:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMove sessions only to the new object store \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e and team are working on, while the rest of the cached objects stay on the current Redis infrastructure\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMake a decision whether and how to move other object types to the same or different object store later?\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThe issue is that even for the main stash we need a datastore that can be written from both datacenters; redis doesn't offer that.\u003c\/p\u003e\n\n\u003cp\u003eThe path forward could be:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMove the session storage to the new datastore, and see how it performs\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMake a call wether to move the mainstash to another instance of the same datastore, or to something else like a multi-dc mcrouter configuration.\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_289\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4895656\" id=\"4895656\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_288\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003eKrinkle\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/2923\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eLimbo\u003c\/a\u003e to \u003ca href=\"\/project\/board\/2923\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003eWatching\u003c\/a\u003e on the \u003ca href=\"\/project\/view\/2923\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003ePerformance-Team (Radar)\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4895656\" data-sigil=\"has-tooltip\" data-meta=\"0_287\"\u003e\u003cspan class=\"screen-only\"\u003eJan 21 2019, 2:21 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-21 02:21:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_292\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2rlpinqyz4xsjmfbcpuo\/PHID-FILE-c2cblouyyjoo7fapx3ek\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Phabricator_maintenance\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4912809\" id=\"4912809\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Phabricator_maintenance\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Phabricator_maintenance\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1025\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1025\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eAcknowledged\u003c\/a\u003e on the \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003eSRE\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4912809\" data-sigil=\"has-tooltip\" data-meta=\"0_290\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2019, 10:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-26 22:54:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_301\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4932035\" id=\"4932035\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003eEvanProdromou\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4932035\" data-sigil=\"has-tooltip\" data-meta=\"0_299\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 4:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 16:26:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_297\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI did some analysis of how we're using ObjectCache in MW core. It seems like we've only got a few calls to the main stash in core right now:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003esite stats (# of users, edits, pages, articles, images), which seems to be persistent\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003erecent (<24h) page deletion flag\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eupload status (24h TTL)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI'd spitball that the first two need multi-DC, and the last one probably doesn't (although it wouldn't hurt).\u003c\/p\u003e\n\n\u003cp\u003eMy understanding was that there's *a lot* of non-session data in the redis store, though. Like, more than the session data. Can we confirm that? Is there a way to get a random sample of the keys in redis so we can analyse where they're coming from?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_304\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/zirzntnbnbimsxdels47\/PHID-FILE-tu3ola3spvpugy5kzs6b\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Mholloway\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4932141\" id=\"4932141\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Mholloway\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Mholloway\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4932141\" data-sigil=\"has-tooltip\" data-meta=\"0_302\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 4:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 16:45:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_313\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4932144\" id=\"4932144\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003eEvanProdromou\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4932144\" data-sigil=\"has-tooltip\" data-meta=\"0_311\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 4:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 16:45:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_309\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eActually, it looks like we've got some extensions using the MainStash, too. I'll dig further into it.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_322\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4934948\" id=\"4934948\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eEvanProdromou\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4934948\" data-sigil=\"has-tooltip\" data-meta=\"0_320\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 15:07:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_318\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSo, re-reading \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T211721#4818580\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_14\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T211721#4818580\u003c\/span\u003e\u003c\/a\u003e and looking at code, it seems like Echo notifications are the big culprit.\u003c\/p\u003e\n\n\u003cp\u003eI've been trying to track our cache use here:\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/www.mediawiki.org\/wiki\/User:EvanProdromou\/ObjectCache_use\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/www.mediawiki.org\/wiki\/User:EvanProdromou\/ObjectCache_use\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eI see a few ways forward, in order of difficulty:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOpen tickets for core code and extensions that are using the ObjectStash "wrong" so that they use the WANCache instead\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOpen tickets for core code and extensions to have configuration options to allow config-time routing of storage requests (echo seen flags go here, TOR exit node fetch status goes there, ...)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eConsider using some kind of namespacing in keys (session, echo, tor, confirmedit, ...) that could be used for routing at run time\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_331\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4935629\" id=\"4935629\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003eEvanProdromou\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4935629\" data-sigil=\"has-tooltip\" data-meta=\"0_329\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 5:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 17:52:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_327\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOne thing we'd need to make sure of is that the Session Storage API isn't designed to be a general-purpose key-value store. \u003ca href=\"https:\/\/www.mediawiki.org\/wiki\/Topic:Uoge9eeznt3nntm0\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eBrad covered it pretty well here\u003c\/a\u003e. I think the primary feature that we use a lot in MW core and extensions is atomic increment, which is not important for sessions but pretty important for stats, counters, toggles, etc.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_334\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4935643\" id=\"4935643\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_333\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003eelukey\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4935643\" data-sigil=\"has-tooltip\" data-meta=\"0_332\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 5:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 17:53:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_345\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_343\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_344\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4937322\" id=\"4937322\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_342\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4937322\" data-sigil=\"has-tooltip\" data-meta=\"0_341\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2019, 7:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-08 07:09:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_339\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_340\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4932035\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4932035\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@EvanProdromou\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI did some analysis of how we're using ObjectCache in MW core. It seems like we've only got a few calls to the main stash in core right now:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003esite stats (# of users, edits, pages, articles, images), which seems to be persistent\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003erecent (<24h) page deletion flag\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eupload status (24h TTL)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI'd spitball that the first two need multi-DC, and the last one probably doesn't (although it wouldn't hurt).\u003c\/p\u003e\n\n\u003cp\u003eMy understanding was that there's *a lot* of non-session data in the redis store, though. Like, more than the session data. Can we confirm that? Is there a way to get a random sample of the keys in redis so we can analyse where they're coming from?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIt's as easy as connecting to the \u003ctt class=\"remarkup-monospaced\"\u003emc*\u003c\/tt\u003e redis instances in the inactive datacenter and looking at the output of \u003ctt class=\"remarkup-monospaced\"\u003eKEYS *\u003c\/tt\u003e. Someone from my team can help gather stats, but yes, session data is a *tiny fraction* of both the data and the read\/write requests to those redises.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_356\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_354\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_355\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4937325\" id=\"4937325\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_353\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4937325\" data-sigil=\"has-tooltip\" data-meta=\"0_352\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2019, 7:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-08 07:17:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_350\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_351\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T212129#4935629\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT212129#4935629\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@EvanProdromou\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eOne thing we'd need to make sure of is that the Session Storage API isn't designed to be a general-purpose key-value store. \u003ca href=\"https:\/\/www.mediawiki.org\/wiki\/Topic:Uoge9eeznt3nntm0\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eBrad covered it pretty well here\u003c\/a\u003e. I think the primary feature that we use a lot in MW core and extensions is atomic increment, which is not important for sessions but pretty important for stats, counters, toggles, etc.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eActually, that's the way it was designed, and it makes sense for session storage and for many other kind of usages.\u003c\/p\u003e\n\n\u003cp\u003eI don't think we should use it for MainStash - we should instead use an mcrouter-based, multi-dc replicated, low-latency ephemeral storage which guarantees we keep the same sub-millisecond latencies we had with redis.\u003c\/p\u003e\n\n\u003cp\u003eCassandra-based storage will always have a latency that's orders of magnitude higher than anything that's purely in-memory; so while I could see a future where we want to have a mid-latency, multi-dc, highly consistent storage for things that need long-term persistence, but that's not how \u003ctt class=\"remarkup-monospaced\"\u003eMainStash\u003c\/tt\u003e works today.\u003c\/p\u003e\n\n\u003cp\u003eOne thing I find quite appalling about this whole situation is that:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThere is no mandate to only use the MW-provided storage interfaces from extensions that run in production\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThere is no track of which of said interfaces every extension uses\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThere is a quite large disconnect between the guarantees that developers might expect from said interfaces and what is really available in production.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThe confusion around this ticket, and in general around moving things out of the redis datastore, seem to all come from this series of deficiencies we should fix together.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_360\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4937362\" id=\"4937362\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_358\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_95\"\u003ejijiki\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/user-jijiki\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003eUser-jijiki\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4937362\" data-sigil=\"has-tooltip\" data-meta=\"0_357\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2019, 7:35 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-08 07:35:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003ejijiki\u003c\/a\u003e subscribed.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_363\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4937708\" id=\"4937708\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003ejijiki\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/3572\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_99\"\u003eInbox \ud83d\udc05\u003c\/a\u003e to \u003ca href=\"\/project\/board\/3572\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003eIn Progress \ud83c\udfcb\ufe0f\u200d\u2640\ufe0f\u003c\/a\u003e on the \u003ca href=\"\/tag\/user-jijiki\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003eUser-jijiki\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4937708\" data-sigil=\"has-tooltip\" data-meta=\"0_361\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2019, 11:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-08 11:00:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_372\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4938421\" id=\"4938421\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003ejijiki\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4938421\" data-sigil=\"has-tooltip\" data-meta=\"0_370\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2019, 3:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-08 15:42:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_368\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_369\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@EvanProdromou\u003c\/span\u003e\u003c\/a\u003e I will try to pull you some stats early next week, probably on Monday :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_382\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4948690\" id=\"4948690\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003ejijiki\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#4948690\" data-sigil=\"has-tooltip\" data-meta=\"0_380\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 12 2019, 6:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-12 18:37:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_378\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@EvanProdromou\u003c\/span\u003e\u003c\/a\u003e After some digging in mc20* redis (codfw), some stats are:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003e409606\u003c\/td\u003e\u003ctd\u003e$wiki:MWSession\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003e18223197\u003c\/td\u003e\u003ctd\u003e$wiki:echo:seen:(alert\/message)\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003e22132316\u003c\/td\u003e\u003ctd\u003eglobal:echo:seen:(alert\/message)\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003e959567\u003c\/td\u003e\u003ctd\u003eglobal:loginnotify:prevSubnet\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003e55002\u003c\/td\u003e\u003ctd\u003eOAUTH:metawiki:nonce (oauth_token)\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003e308605\u003c\/td\u003e\u003ctd\u003eglobal:Wikimedia\\Rdbms\\ChronologyProtector:mtime\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003e45473452\u003c\/td\u003e\u003ctd\u003eTotal keys\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003ePlease let me know if you need anything else:)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_385\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4949790\" id=\"4949790\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_384\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added projects: \u003ca href=\"\/project\/view\/2311\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_107\"\u003eServices (doing)\u003c\/a\u003e, \u003ca href=\"\/tag\/user-mobrovac\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eUser-mobrovac\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4949790\" data-sigil=\"has-tooltip\" data-meta=\"0_383\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 13 2019, 12:57 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-13 00:57:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_396\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_394\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_395\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4950510\" id=\"4950510\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_393\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4950510\" data-sigil=\"has-tooltip\" data-meta=\"0_392\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 13 2019, 11:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-13 11:14:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_390\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_391\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_110\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jijiki\u003c\/span\u003e\u003c\/a\u003e what is the total number of items stored on those redises? So that I can understand how much of that is used by the sessions. I guess less than 1%?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_405\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4951458\" id=\"4951458\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003ejijiki\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4951458\" data-sigil=\"has-tooltip\" data-meta=\"0_403\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 13 2019, 4:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-13 16:25:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_401\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_402\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_112\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e I updated the table above\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_408\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4953742\" id=\"4953742\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_407\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_113\"\u003ejijiki\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/3572\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003eIn Progress \ud83c\udfcb\ufe0f\u200d\u2640\ufe0f\u003c\/a\u003e to \u003ca href=\"\/project\/board\/3572\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_115\"\u003eSt\u003c\/a\u003e on the \u003ca href=\"\/tag\/user-jijiki\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003eUser-jijiki\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4953742\" data-sigil=\"has-tooltip\" data-meta=\"0_406\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2019, 11:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-14 11:00:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_417\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4972422\" id=\"4972422\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_416\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_117\"\u003eEvanProdromou\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4972422\" data-sigil=\"has-tooltip\" data-meta=\"0_415\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 21 2019, 4:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-21 16:43:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_413\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_414\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_118\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks \u003ca href=\"\/p\/jijiki\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jijiki\u003c\/span\u003e\u003c\/a\u003e and \u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eAssuming those stats are representative, I see some key takeaways:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e40M \/ 45M keys (~88%) of keys are for echo. We should probably have a way to configure echo storage explicitly; I don't think that's possible right now. I'll open a ticket for that immediately.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e2 of the top 6 types of keys aren't accounted for in the analysis I did. Looking at, say, ChronologyProtector, I still can't figure out why that's going to the main stash. So, there's probably a lot more code like that.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAbout 3M keys not in this table. That's a long tail of storage. It'd be interesting to see the next 20 or 100 patterns.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eAfter that, I think \u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e has some good points. I wonder if this would do the trick for us:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eStandard way to get a stash for a component or extension which checks if there's a configured stash for that component\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eStandard way to define the contract for the component or extension's stash needs, so if there's none configured, the best-fitting one can be applied\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eStandard way to figure out what component or extension put a key in the stash\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI'm going to follow up on this further.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_426\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kd4xv6kmiqlmbg5i3ynf\/PHID-FILE-q756a47n7hkxmwqviotl\/profile-profile.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aaron\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4992682\" id=\"4992682\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_425\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_119\"\u003eaaron\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4992682\" data-sigil=\"has-tooltip\" data-meta=\"0_424\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 28 2019, 10:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-28 22:01:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_422\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_423\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_120\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChronologyProtector positions should be applied to all DCs to handle cases the sticky DC cookie isn't enough (e.g. redirects to other WMF wiki domains). This is why it uses the main stash.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_429\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/yylm6ryl4fou2honnhxf\/PHID-FILE-fk65uy3t4xsbhfqzo3ew\/profile-10622909_10152249591556697_1835323827866737317_n.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/SBisson\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5138290\" id=\"5138290\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/SBisson\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_121\"\u003eSBisson\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5138290\" data-sigil=\"has-tooltip\" data-meta=\"0_427\"\u003e\u003cspan class=\"screen-only\"\u003eApr 25 2019, 5:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-04-25 17:19:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_433\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/de7rnuucgwsa2u5ow7bn\/PHID-FILE-towcvgcmdefffz27aphc\/profile-gravatar.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Catrope\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5169083\" id=\"5169083\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Catrope\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003eCatrope\u003c\/a\u003e mentioned this in \u003ca href=\"\/T222851\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_123\"\u003eT222851: Improve Echo seentime code for multi-DC access\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5169083\" data-sigil=\"has-tooltip\" data-meta=\"0_430\"\u003e\u003cspan class=\"screen-only\"\u003eMay 8 2019, 11:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-08 23:25:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_432\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Catrope\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003eCatrope\u003c\/a\u003e added a subtask: \u003ca href=\"\/T222851\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003eT222851: Improve Echo seentime code for multi-DC access\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_442\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xfpo7764mzhw24fxdxpi\/PHID-FILE-anrie3fu7jeg4446msm3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/EvanProdromou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5170898\" id=\"5170898\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_441\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/EvanProdromou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003eEvanProdromou\u003c\/a\u003e removed a parent task: \u003ca href=\"\/T206016\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_127\"\u003eT206016: Create a service for session storage\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5170898\" data-sigil=\"has-tooltip\" data-meta=\"0_440\"\u003e\u003cspan class=\"screen-only\"\u003eMay 9 2019, 5:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-09 17:34:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_438\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_439\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_128\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI moved this task so it's a direct child of the active\/active data centre discussion, rather than being tied specifically to session storage.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_445\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/zirzntnbnbimsxdels47\/PHID-FILE-tu3ola3spvpugy5kzs6b\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Mholloway\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5178007\" id=\"5178007\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Mholloway\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_129\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Mholloway\u003c\/a\u003e mentioned this in \u003ca href=\"\/T206504\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003eT206504: Create a new endpoint which returns articles in need of a description\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5178007\" data-sigil=\"has-tooltip\" data-meta=\"0_443\"\u003e\u003cspan class=\"screen-only\"\u003eMay 13 2019, 6:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-13 18:39:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_456\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_454\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5199080\" id=\"5199080\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003eKrinkle\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5199080\" data-sigil=\"has-tooltip\" data-meta=\"0_452\"\u003e\u003cspan class=\"screen-only\"\u003eMay 20 2019, 11:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-20 23:35:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_450\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe official requirements of Stash are in line with the REST-storage service being created for sessions.\u003c\/p\u003e\n\n\u003cp\u003eExpectations of MediaWiki 'Main stash' are (as documented in \u003ca href=\"https:\/\/doc.wikimedia.org\/mediawiki-core\/master\/php\/classObjectCache.html#details\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eObjectCache.php\u003c\/a\u003e):\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eStored in primary DC.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eReplicated to other DCs.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSupport for READ_LATEST exists (e.g. master query) are possible, but should only happen in POST and Job context (discouraged over GET).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e"Fairly strong persistence" (for third-parties it defaults to MySQL, at WMF we use Redis, and LRU evictions may happen but overall unpopular keys are expected to last fairly long, and the store is a good alternative to custom database tables for data that has many keys, is expensive to compute, or data that comes from an off-wiki source).\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eFor any individual uses of the stash that can be changed to not need all these requirements and\/or that didn't need it in the first place, can of course be migrated whenever. If we have limited storage capacity and expect to be unable to house current data, we may even want to require some of these to migrate first.\u003c\/p\u003e\n\n\u003cp\u003eHowever, I do not think that "stash" as an abstract interface should be changed to something other than the above requirements. As such, assuming no unsupported raw use of these Redis servers, they could be decom'ed without much controversy I expect. Basically, as soon as MW config for "stash" switched from Redis to Cassandra.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_459\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5199081\" id=\"5199081\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_133\"\u003eKrinkle\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-gksmxns4vqnc3kv\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5199081\" data-sigil=\"has-tooltip\" data-meta=\"0_457\"\u003e\u003cspan class=\"screen-only\"\u003eMay 20 2019, 11:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-20 23:36:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-o7j6u4p5uj2t757llz6a"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-k5fovjqtabvxirzd6sjr"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-sncfzmpwbqcootc"},{"hovercardSpec":{"objectPHID":"PHID-USER-5dqihbanu3caaj7pigif"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jzttt2rzt4bpbzkerkod"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-cm6l3muxcwmitvpzcc4u"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-4dauou5gzevey3xwayrt"}},{"phid":"PHID-XACT-TASK-q6hxjj4gq466wxu"},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-f2ftqhrflqtsulq"},{"phid":"PHID-XACT-TASK-7oxibz5gam3rtkg"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-rscluxigsugkibt"},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"phid":"PHID-XACT-TASK-wpqmjj36xwtz6d7"},{"phid":"PHID-XACT-TASK-uag6izjzm52t7dg"},{"hovercardSpec":{"objectPHID":"PHID-USER-xwxce3kpvsv54t735g7q"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-ftnyalpk35x4hwh"},{"phid":"PHID-XACT-TASK-v4sreu5u3mozi57"},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6b6w5qogf4deywgbkuzs"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-6za5vklmpbqwg5g"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-duoymglazrdcr3v7bxzc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-y7k6c7zwrcb62hzgiulu"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-d2s4hyfykrv3r6lt5txg"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-2b7oz62ylk3jk4aus262"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-a66mfxgb6yhoyqgrges6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-2b7oz62ylk3jk4aus262"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-dd55vyugvc2wtnguzbso"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-lcjmcdvcozeqvfm"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-pe6x4ydcw7bhypk"},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-27kdwzvrc74fzrnjl6sr"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-pfjzrm3k6t27kjadirze"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-cm6l3muxcwmitvpzcc4u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xragmpydvtwulk3zv4o7"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-c4yrc5wlq7aalby54odc"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-knvdos3w6r5kdmycbgir"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-biy5xgnrgya2a67"},{"hovercardSpec":{"objectPHID":"PHID-USER-d2tiurktk2jgxzcpd2jw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-6bp7kawr3r7crkx"},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-qcwybzkoeddzxfq"},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-uuyfi6e2tfmsj3t"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-hyn4c5nfb7lhg6p"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-tuvmufnf3bq24sy"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-l7i5j2feyc5mnrmmxnjg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-h4bctpqri3johntxndqa"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-qh6ktbibca3mazfwbx3k"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-l7i5j2feyc5mnrmmxnjg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"phid":"PHID-XACT-TASK-ylm5xkmtcwxcrn6"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"phid":"PHID-XACT-TASK-u474jkb3b5pnvmb"},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-bdbgsen5dh2gw4qfg4dx"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-4w5ynwe7i367iiu4lalr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-szu6gfwubzxozmu"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"phid":"PHID-XACT-TASK-guzgr7he3bzqeuq"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-qh6ktbibca3mazfwbx3k"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-r5rcwztanbv7or6sotvs"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-l7i5j2feyc5mnrmmxnjg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"phid":"PHID-XACT-TASK-552cfvg7renjh6z"},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"phid":"PHID-XACT-TASK-zcna6v5v7eyoqj3"},{"hovercardSpec":{"objectPHID":"PHID-USER-rmj3kg6svipmzza2iogf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fovtl67ew4l4cc3oeypc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-qasesmkth4xw6ue5af6s"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fovtl67ew4l4cc3oeypc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-qasesmkth4xw6ue5af6s"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2e3ss3dyx27wu7wkk3qw"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-duoymglazrdcr3v7bxzc"}},{"phid":"PHID-XACT-TASK-r7b22cz6oyzv6xw"},{"hovercardSpec":{"objectPHID":"PHID-USER-d2tiurktk2jgxzcpd2jw"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yxzy7yiazvgfnckeaybq"}},{"phid":"PHID-XACT-TASK-iyshokn3a7yfla4"},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7ed6dfdywfluxw6","anchor":"4827944"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-xfvdpk7xnv4nucz","anchor":"4827955"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-brqgf3mcyimtiyw","anchor":"4830437"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4stxhuxr2vyyfau","anchor":"4831752"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-sncfzmpwbqcootc\/","ref":"T212129#4831754"},[],{"anchor":"4831754"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_146\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sncfzmpwbqcootc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_148\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-sncfzmpwbqcootc","anchor":"4831754"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-daa4gdliz4avvln","anchor":"4832235"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-esdcwou5khb3yob","anchor":"4832383"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-q6hxjj4gq466wxu\/","ref":"T212129#4832391"},[],{"anchor":"4832391"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_163\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-q6hxjj4gq466wxu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_165\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-q6hxjj4gq466wxu","anchor":"4832391"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-f2ftqhrflqtsulq\/","ref":"T212129#4832455"},[],{"anchor":"4832455"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-f2ftqhrflqtsulq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_174\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-f2ftqhrflqtsulq","anchor":"4832455"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7oxibz5gam3rtkg\/","ref":"T212129#4832565"},[],{"anchor":"4832565"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_183\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7oxibz5gam3rtkg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_185\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7oxibz5gam3rtkg","anchor":"4832565"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rscluxigsugkibt\/","ref":"T212129#4832587"},[],{"anchor":"4832587"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_192\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_193\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rscluxigsugkibt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_194\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rscluxigsugkibt","anchor":"4832587"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wpqmjj36xwtz6d7\/","ref":"T212129#4833043"},[],{"anchor":"4833043"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_203\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_204\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wpqmjj36xwtz6d7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_205\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_206\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wpqmjj36xwtz6d7","anchor":"4833043"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uag6izjzm52t7dg\/","ref":"T212129#4833375"},[],{"anchor":"4833375"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_212\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_213\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uag6izjzm52t7dg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_214\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_215\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-uag6izjzm52t7dg","anchor":"4833375"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ftnyalpk35x4hwh\/","ref":"T212129#4833420"},[],{"anchor":"4833420"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_225\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_226\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ftnyalpk35x4hwh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_227\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ftnyalpk35x4hwh","anchor":"4833420"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-v4sreu5u3mozi57\/","ref":"T212129#4834330"},[],{"anchor":"4834330"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_236\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v4sreu5u3mozi57\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_238\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-v4sreu5u3mozi57","anchor":"4834330"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yiej2xbciessymq","anchor":"4834491"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6za5vklmpbqwg5g\/","ref":"T212129#4835375"},[],{"anchor":"4835375"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_248\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6za5vklmpbqwg5g\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_250\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6za5vklmpbqwg5g","anchor":"4835375"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vtdzdq466mnz2qy","anchor":"4870031"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-dzzfhqrln5uivhs","anchor":"4881160"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-zqly4kgrycucvpk","anchor":"4881168"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lcjmcdvcozeqvfm\/","ref":"T212129#4885223"},[],{"anchor":"4885223"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_267\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_268\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lcjmcdvcozeqvfm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_269\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_270\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lcjmcdvcozeqvfm","anchor":"4885223"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pe6x4ydcw7bhypk\/","ref":"T212129#4885245"},[],{"anchor":"4885245"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_276\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_277\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pe6x4ydcw7bhypk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_278\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_279\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-pe6x4ydcw7bhypk","anchor":"4885245"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hiuf5ckc546m2me","anchor":"4895656"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xovlupeoyzs3unw","anchor":"4912809"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-biy5xgnrgya2a67\/","ref":"T212129#4932035"},[],{"anchor":"4932035"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_293\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-biy5xgnrgya2a67\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_295\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-biy5xgnrgya2a67","anchor":"4932035"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qhysfddllv2ckgj","anchor":"4932141"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6bp7kawr3r7crkx\/","ref":"T212129#4932144"},[],{"anchor":"4932144"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_305\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_306\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6bp7kawr3r7crkx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_307\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6bp7kawr3r7crkx","anchor":"4932144"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qcwybzkoeddzxfq\/","ref":"T212129#4934948"},[],{"anchor":"4934948"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_314\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_315\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qcwybzkoeddzxfq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_316\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qcwybzkoeddzxfq","anchor":"4934948"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uuyfi6e2tfmsj3t\/","ref":"T212129#4935629"},[],{"anchor":"4935629"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_323\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_324\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uuyfi6e2tfmsj3t\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_325\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uuyfi6e2tfmsj3t","anchor":"4935629"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-c7hewq4umvko3za","anchor":"4935643"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hyn4c5nfb7lhg6p\/","ref":"T212129#4937322"},[],{"anchor":"4937322"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_335\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hyn4c5nfb7lhg6p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_337\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_338\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-hyn4c5nfb7lhg6p","anchor":"4937322"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tuvmufnf3bq24sy\/","ref":"T212129#4937325"},[],{"anchor":"4937325"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_346\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_347\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tuvmufnf3bq24sy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_348\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_349\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tuvmufnf3bq24sy","anchor":"4937325"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-2ecveluakqai544","anchor":"4937362"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-baxctekpfckbur3","anchor":"4937708"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ylm5xkmtcwxcrn6\/","ref":"T212129#4938421"},[],{"anchor":"4938421"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_364\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ylm5xkmtcwxcrn6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_366\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_367\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ylm5xkmtcwxcrn6","anchor":"4938421"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u474jkb3b5pnvmb\/","ref":"T212129#4948690"},[],{"anchor":"4948690"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_373\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u474jkb3b5pnvmb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_375\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_376\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-u474jkb3b5pnvmb\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u474jkb3b5pnvmb","anchor":"4948690"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-f6iwxkzpvb6uz3e","anchor":"4949790"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-szu6gfwubzxozmu\/","ref":"T212129#4950510"},[],{"anchor":"4950510"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_386\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_387\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-szu6gfwubzxozmu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_388\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_389\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-szu6gfwubzxozmu","anchor":"4950510"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-guzgr7he3bzqeuq\/","ref":"T212129#4951458"},[],{"anchor":"4951458"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_397\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_398\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-guzgr7he3bzqeuq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_399\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_400\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-guzgr7he3bzqeuq","anchor":"4951458"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ccgrhcxiakf6eb3","anchor":"4953742"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-552cfvg7renjh6z\/","ref":"T212129#4972422"},[],{"anchor":"4972422"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_409\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-552cfvg7renjh6z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_411\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_412\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-552cfvg7renjh6z","anchor":"4972422"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zcna6v5v7eyoqj3\/","ref":"T212129#4992682"},[],{"anchor":"4992682"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_418\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zcna6v5v7eyoqj3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_420\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_421\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-zcna6v5v7eyoqj3","anchor":"4992682"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-q3li6htsnsjz5yh","anchor":"5138290"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-ieuijtsxxce46ln","anchor":"5169083"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-r7b22cz6oyzv6xw\/","ref":"T212129#5170898"},[],{"anchor":"5170898"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_434\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r7b22cz6oyzv6xw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_436\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sffxat6dw36xujs","anchor":"5170898"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2arslmmgrqxxmxa","anchor":"5178007"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-iyshokn3a7yfla4\/","ref":"T212129#5199080"},[],{"anchor":"5199080"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_446\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_447\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-iyshokn3a7yfla4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_448\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-iyshokn3a7yfla4","anchor":"5199080"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gksmxns4vqnc3kv","anchor":"5199081"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}