for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4784903\" id=\"4784903\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_47\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4784903\" data-sigil=\"has-tooltip\" data-meta=\"0_46\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 1:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 13:20:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4784912\" id=\"4784912\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_50\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_11\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/2878928\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4784912\" data-sigil=\"has-tooltip\" data-meta=\"0_49\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 1:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 13:20:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_60\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_61\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4784930\" id=\"4784930\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_59\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003ejcrespo\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003eDBA\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4784930\" data-sigil=\"has-tooltip\" data-meta=\"0_58\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 1:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 13:25:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_56\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_57\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_15\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNot sure if to file this separately or not, but:\u003c\/p\u003e\n\n\u003cp\u003ewe also purchased an extra host this time- We can make it work transparently through some kind of proxying (not sure about that), but maybe it makes sense to some kind of service failover automatically, so we don't have a SPOF on each host?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ms665yc73j6nadjitytk\/PHID-FILE-mocohumlrcbe2lcuplml\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Anomie\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_71\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_72\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785179\" id=\"4785179\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_70\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Anomie\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003eAnomie\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785179\" data-sigil=\"has-tooltip\" data-meta=\"0_69\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 2:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 14:52:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_67\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_68\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_17\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eThose first column of IPs (10.64.0.12, 10.64.32.72, 10.64.48.128) are not really IPs but sharding keys.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eUgh, that's awful.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWill that generate a miss as soon as they get changed in config?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI think it'd be the same thing as if you were to change the identifiers of all the memcached servers (we shard those too, don't we?). Some keys would still be assigned to the same server, while the majority would wind up switching to a different one.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWill that break something else mediawiki-wise?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI don't see anything else that would be broken by it. As far as I can tell the tag is only used to pick a server.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eThe old keys would need to be purged manually after $retention_period probably?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIt looks like purging of old keys relies only on the stored \u003ctt class=\"remarkup-monospaced\"\u003eexptime\u003c\/tt\u003e. I'd guess that \u003ctt class=\"remarkup-monospaced\"\u003eexptime\u003c\/tt\u003e is set based on the retention period, so I'd guess that manual purging wouldn't be needed. But I could be wrong.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_82\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_83\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785209\" id=\"4785209\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785209\" data-sigil=\"has-tooltip\" data-meta=\"0_80\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 14:57:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_78\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_19\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eUgh, that's awful.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eTo clarify the context- this was done in a rush with a record patch the day before the first datacenter switch, so let's praise this was doable at all by the great Performance team! Those made some sense at the time as the sharding key were the real ip adresses, before codfw ever existed.\u003c\/p\u003e\n\n\u003cp\u003eHowever, we need a way to move forward- and we may now have the extra servers to do so...\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_93\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785284\" id=\"4785284\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_92\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003ejcrespo\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/wikimedia-performance-publish\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003eWikimedia-Performance-publish\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785284\" data-sigil=\"has-tooltip\" data-meta=\"0_91\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 15:07:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_89\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_22\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis is not on performance, they may be able to give some ideas on how to move forward while minimizing impact to hit ratio, as they helped setting it up.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_98\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4785286\" id=\"4785286\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_97\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_24\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a project: \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003ePerformance-Team\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/2879118\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785286\" data-sigil=\"has-tooltip\" data-meta=\"0_96\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 15:07:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_101\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4785291\" id=\"4785291\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_100\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003ejcrespo\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/wikimedia-performance-publish\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003eWikimedia-Performance-publish\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785291\" data-sigil=\"has-tooltip\" data-meta=\"0_99\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 3:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 15:08:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_114\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_110\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_112\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_113\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785416\" id=\"4785416\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785416\" data-sigil=\"has-tooltip\" data-meta=\"0_108\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 3:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 15:30:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_106\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_29\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSo given the fact that we started with all the pc partially cold after switching back from codfw to eqiad (due to replication being disconnected) and we kinda survived - maybe we can try to switch keys one by one on a controlled manner?. So like flushing one parsercache per month?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i3mfqczbjenhdwyj5h44\/PHID-FILE-f5l7mmlyhoyqwhe43sne\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Banyek\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785435\" id=\"4785435\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Banyek\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Banyek\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785435\" data-sigil=\"has-tooltip\" data-meta=\"0_121\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 3:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 15:34:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_119\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T210725#4785416\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT210725#4785416\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eSo given the fact that we started with all the pc partially cold after switching back from codfw to eqiad (due to replication being disconnected) and we kinda survived - maybe we can try to switch keys one by one on a controlled manner?. So like flushing one parsercache per month?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI like the idea - we should ensure that the services undisturbed even in the worst conditions - if we flush those regularly we'll be sure that we don't have to worry about a crash\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_126\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4785469\" id=\"4785469\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003eTriage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eBacklog\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003eDBA\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785469\" data-sigil=\"has-tooltip\" data-meta=\"0_124\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 3:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 15:48:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rus3onc75davarls6nbc\/PHID-FILE-d4tltleyv665iw455lvp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Imarlier\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785537\" id=\"4785537\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Imarlier\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Imarlier\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785537\" data-sigil=\"has-tooltip\" data-meta=\"0_133\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 4:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 16:07:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_131\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_36\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T210725#4785435\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT210725#4785435\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Banyek\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Banyek\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T210725#4785416\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT210725#4785416\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eSo given the fact that we started with all the pc partially cold after switching back from codfw to eqiad (due to replication being disconnected) and we kinda survived - maybe we can try to switch keys one by one on a controlled manner?. So like flushing one parsercache per month?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI like the idea - we should ensure that the services undisturbed even in the worst conditions - if we flush those regularly we'll be sure that we don't have to worry about a crash\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e+100 -- if we can't survive one of the parsercache machines being flushed, then we're putting ourselves in quite a risky state given the potential of hardware failing.\u003c\/p\u003e\n\n\u003cp\u003eIt looks like it took \u003ca href=\"https:\/\/grafana.wikimedia.org\/dashboard\/db\/parser-cache?orgId=1&from=1539129600000&to=1539734400000\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eabout 36 hours\u003c\/a\u003e for hit rates to recover to basically normal levels after the codfw->eqiad switch back in October.  With normal hit rates around 68%, we should expect overall hit rate to drop to about 46% initially.  After 24 hours, hit rate should be about 57%, and will slowly rise back to normal levels over the next 48 hours.\u003c\/p\u003e\n\n\u003cp\u003eThis shouldn't be any issue.  Based on \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Incident_documentation\/20181016-eqiad_parsercache_empty_post-switchover\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethe codfw-eqiad switchover incident documentation\u003c\/a\u003e, we started to see recovery when hit rates got up to 20-25% (recovery was around 15:00, and the hit rate during the 15:00 hour was 25%).  By the time we were up to around 45% hit rates there were no further effects being discussed.\u003c\/p\u003e\n\n\u003cp\u003e(Also worth noting is that 23% is about the hit rate we should expect to see if we changed all three shard keys at the same time, meaning that we could likely make this shift at once.  Not suggesting it, just pointing it out :-)\u003c\/p\u003e\n\n\u003cp\u003eSomething that I would suggest, if it's possible, is taking this opportunity to expand the parsercache pool by assigning each host multiple times:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e$wmgParserCacheDBs = [\n    'pc1'   => '10.192.0.104',  # pc2007, A1 4.4TB 256GB\n    'pc2'   => '10.192.0.104',  # pc2007, A1 4.4TB 256GB\n    'pc3'   => '10.192.0.104',  # pc2007, A1 4.4TB 256GB\n    'pc4'   => '10.192.0.104',  # pc2007, A1 4.4TB 256GB\n    'pc5'  => '10.192.16.35',  # pc2008, B3 4.4TB 256GB\n    'pc6'  => '10.192.16.35',  # pc2008, B3 4.4TB 256GB\n    'pc7'  => '10.192.16.35',  # pc2008, B3 4.4TB 256GB\n    'pc8'  => '10.192.16.35',  # pc2008, B3 4.4TB 256GB\n    'pc9' => '10.192.32.10',  # pc2009, C1 4.4TB 256GB\n    'pc10' => '10.192.32.10',  # pc2009, C1 4.4TB 256GB\n    'pc11' => '10.192.32.10',  # pc2009, C1 4.4TB 256GB\n    'pc12' => '10.192.32.10',  # pc2009, C1 4.4TB 256GB\n    # 'spare' => '10.192.48.14',  # pc2010, D3 4.4TB 256GB # spare host. Use it to replace any of the above if needed\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThis would, potentially require other adjustments, like connection pool sizes on the app side and max_connections on the db side.  The advantage that it has is that it allows for more flexibility in the future.  Given our natural hit rate, moving a single cache shard would only reduce hit rate from 68% to about 63%, we could integrate a fourth parsercache host easily, and so on.  Not something that's really necessary, but maybe worth thinking about as long as we're here.\u003c\/p\u003e\n\n\u003cp\u003e(My example has 12 shards because of the even divisability with both 3 and 4, I'd think about going with either 12 or 24 if you do decide to increase.)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_146\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785678\" id=\"4785678\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785678\" data-sigil=\"has-tooltip\" data-meta=\"0_142\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 4:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 16:39:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_140\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_141\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe multiple keys is ok, we already do that for the External Storage, my worry is not that, as much as the migration method (which is not get affected negatively or positively by it, even if it is desirable for other reasons).\u003c\/p\u003e\n\n\u003cp\u003eFor what I understood from your proposals + my own take, the best proposal is to just do the change (maybe on a low-traffic time) and get hit by the lower hit ratio (and probably there is not other easy way) and brace for the (hopefully small) period of bad hit ratio.\u003c\/p\u003e\n\n\u003cp\u003eRegarding the high availability, I was thinking of setting up the 4th host empty so we can lose the content (but not the service) when failing over- if possibly, automtically. Is that something that would be easy (or even desirable) to implement on mediawiki logic (the same way it works on the metadata or data hosts), or should we use a proxy? Alternatively, we can setup 4 hosts actively, but I think the sharding function won't work ok?\u003c\/p\u003e\n\n\u003cp\u003eI don't need you to start any project, just want your opinions on this for planing porpuses before doing the above architectural changes- aiming for both good performance AND automatic failure handling (unlike the previous state, with 3 separate SPOF).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_157\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ms665yc73j6nadjitytk\/PHID-FILE-mocohumlrcbe2lcuplml\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Anomie\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_155\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785845\" id=\"4785845\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_154\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Anomie\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eAnomie\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4785845\" data-sigil=\"has-tooltip\" data-meta=\"0_153\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 5:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 17:08:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_151\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T210725#4785678\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT210725#4785678\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAlternatively, we can setup 4 hosts actively, but I think the sharding function won't work ok?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThe sharding function is pretty simple: it just hashes the tag+key for each host's tag, and whichever tag results in the lowest hash is the one that's used for that key.\u003c\/p\u003e\n\n\u003cp\u003eThat seems like it should behave ok when adding a 4th host: about 25% of keys that would have been hits will be misses because they're now sharded to the new host, and each of the existing hosts would have about 25% of their keys become "dead". Dropping from 4 to 3 hosts should work ok too: the dropped host would have been handling 25% of the keys, which will be evenly spread over the 3 remaining hosts.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_167\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4785934\" id=\"4785934\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#4785934\" data-sigil=\"has-tooltip\" data-meta=\"0_165\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 5:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 17:38:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_163\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eAlternatively, we can setup 4 hosts actively, but I think the sharding function won't work ok?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSorry, my fault, I didn't express well (or maybe it is me who is wrong and didn't understood). Would those changes happen dynamically if , eg, 'pc2' stops responding while being configured? Or maybe for that to happem, we have to setup the 4 severs in 6 entries, 3 with pc1-3, and the 4th 3 times with the same ones? I don't know if you understand what I want to do (automatic service switch on unresponsive db).\u003c\/p\u003e\n\n\u003cp\u003eWhat we want to avoid is being paged at 4am if a server fails -having to do manual changes :-)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ms665yc73j6nadjitytk\/PHID-FILE-mocohumlrcbe2lcuplml\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Anomie\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/5\/\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4786136\" id=\"4786136\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Anomie\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eAnomie\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4786136\" data-sigil=\"has-tooltip\" data-meta=\"0_176\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2018, 6:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-11-29 18:11:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_174\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOn the MediaWiki site the relevant code is in SqlBagOStuff. It looks like the current code will treat a failing server as a miss and will drop writes, I see no logic for automatic switching.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-5bt5njxlaxwmf35inp32","contextPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jjhmunq2sjerxgggmdhq"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"phid":"PHID-XACT-TASK-qzy7tjutocahpab"},{"hovercardSpec":{"objectPHID":"PHID-USER-uqcn2l4ng4murmyfnvyp"}},{"phid":"PHID-XACT-TASK-2q3ny4fvdw7us3p"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-tccwyikq6kqit42"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-fb65k3l2a7dy4zcehv7v"}},{"phid":"PHID-XACT-TASK-kmg76c5h7ld2wve"},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-fb65k3l2a7dy4zcehv7v"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-yg7eebxzwvbzffm"},{"hovercardSpec":{"objectPHID":"PHID-USER-5bt5njxlaxwmf35inp32"}},{"phid":"PHID-XACT-TASK-ovdold257sdhdtl"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bk6sgdzdhctxupi3vn5b"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-54xjtvb6c5vxcsr2izkm"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"phid":"PHID-XACT-TASK-bh2phkitac6bfwl"},{"hovercardSpec":{"objectPHID":"PHID-USER-jzttt2rzt4bpbzkerkod"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-7age7j6tyvseh6p"},{"hovercardSpec":{"objectPHID":"PHID-USER-uqcn2l4ng4murmyfnvyp"}},{"phid":"PHID-XACT-TASK-5ktwu552ysopk3g"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-escis5aoimnstqk"},{"hovercardSpec":{"objectPHID":"PHID-USER-uqcn2l4ng4murmyfnvyp"}},{"phid":"PHID-XACT-TASK-mpaoj7yz7ksuwk4"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cu2khnzucd6w3yd","anchor":"4784903"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-ueyqpmzb2jts7yg","anchor":"4784912"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qzy7tjutocahpab\/","ref":"T210725#4784930"},[],{"anchor":"4784930"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_52\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_53\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qzy7tjutocahpab\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_54\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_55\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-5ryihpnqrlpvxa3","anchor":"4784930"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2q3ny4fvdw7us3p\/","ref":"T210725#4785179"},[],{"anchor":"4785179"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_63\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_64\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2q3ny4fvdw7us3p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_65\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_66\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-2q3ny4fvdw7us3p","anchor":"4785179"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tccwyikq6kqit42\/","ref":"T210725#4785209"},[],{"anchor":"4785209"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_74\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_75\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tccwyikq6kqit42\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_76\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_77\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tccwyikq6kqit42","anchor":"4785209"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kmg76c5h7ld2wve\/","ref":"T210725#4785284"},[],{"anchor":"4785284"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_85\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_86\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kmg76c5h7ld2wve\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_87\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ibko5pz6grbqss4","anchor":"4785284"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-u37n5yy57gzyy2k","anchor":"4785286"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dpq5gyrdvgsw4tv","anchor":"4785291"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yg7eebxzwvbzffm\/","ref":"T210725#4785416"},[],{"anchor":"4785416"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_102\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yg7eebxzwvbzffm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_104\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-yg7eebxzwvbzffm","anchor":"4785416"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ovdold257sdhdtl\/","ref":"T210725#4785435"},[],{"anchor":"4785435"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_115\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ovdold257sdhdtl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ovdold257sdhdtl","anchor":"4785435"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xrmebi6ha6oj4dt","anchor":"4785469"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bh2phkitac6bfwl\/","ref":"T210725#4785537"},[],{"anchor":"4785537"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_127\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_128\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bh2phkitac6bfwl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_129\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bh2phkitac6bfwl","anchor":"4785537"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7age7j6tyvseh6p\/","ref":"T210725#4785678"},[],{"anchor":"4785678"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_136\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_137\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7age7j6tyvseh6p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_138\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_139\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-7age7j6tyvseh6p","anchor":"4785678"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5ktwu552ysopk3g\/","ref":"T210725#4785845"},[],{"anchor":"4785845"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_147\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_148\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5ktwu552ysopk3g\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_149\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_150\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-5ktwu552ysopk3g","anchor":"4785845"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-escis5aoimnstqk\/","ref":"T210725#4785934"},[],{"anchor":"4785934"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_158\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-escis5aoimnstqk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_160\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-escis5aoimnstqk\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-escis5aoimnstqk","anchor":"4785934"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mpaoj7yz7ksuwk4\/","ref":"T210725#4786136"},[],{"anchor":"4786136"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_170\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mpaoj7yz7ksuwk4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Backport Deployer","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-mpaoj7yz7ksuwk4","anchor":"4786136"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}