for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1936342\" id=\"1936342\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_48\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003efgiunchedi\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1936342\" data-sigil=\"has-tooltip\" data-meta=\"0_47\"\u003e\u003cspan class=\"screen-only\"\u003eJan 15 2016, 10:54 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-01-15 10:54:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_49\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003efgiunchedi\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003e\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_50\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003efgiunchedi\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-jhyktdezhb4cuzq\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_51\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003efgiunchedi\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003eSRE\u003c\/a\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_52\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003efgiunchedi\u003c\/a\u003e added subscribers: \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003efgiunchedi\u003c\/a\u003e, \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003eAklapper\u003c\/a\u003e, \u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003eDzahn\u003c\/a\u003e, \u003ca href=\"\/p\/StudiesWorld\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003eStudiesWorld\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_56\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1962208\" id=\"1962208\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_55\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003eDzahn\u003c\/a\u003e assigned this task to \u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\"\u003eelukey\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1962208\" data-sigil=\"has-tooltip\" data-meta=\"0_54\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2016, 5:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-01-25 17:34:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1963069\" id=\"1963069\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_64\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1963069\" data-sigil=\"has-tooltip\" data-meta=\"0_63\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2016, 7:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-01-25 19:49:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_61\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_62\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_21\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_1\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@elukey\u003c\/span\u003e\u003c\/a\u003e per chat in the ops meeting. ask me anytime about where to start to get a serial console, use installserver etc.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_76\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_74\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_75\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1965105\" id=\"1965105\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_73\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003eJoe\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1965105\" data-sigil=\"has-tooltip\" data-meta=\"0_72\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2016, 9:07 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-01-26 09:07:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_70\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_71\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_22\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAll memcached hosts have both memcached and the session-related redis.\u003c\/p\u003e\n\n\u003cp\u003eSo reinstalling them has a small but non-trivial effect: when a server goes down, we lose 1\/18th of the current user sessions.\u003c\/p\u003e\n\n\u003cp\u003eAlso, mc1001-1003 are also used as lock manager redises, so when reinstalling those a special care should be taken so that we change operations\/mediawiki-config first.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1972487\" id=\"1972487\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1972487\" data-sigil=\"has-tooltip\" data-meta=\"0_83\"\u003e\u003cspan class=\"screen-only\"\u003eJan 28 2016, 12:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-01-28 00:49:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_81\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_82\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003elist of servers:\u003c\/p\u003e\n\n\u003cp\u003emc1001.eqiad.wmnet: True\u003cbr \/\u003e\nmc1002.eqiad.wmnet: True\u003cbr \/\u003e\nmc1003.eqiad.wmnet: True\u003cbr \/\u003e\nmc1004.eqiad.wmnet: True\u003cbr \/\u003e\nmc1005.eqiad.wmnet: True\u003cbr \/\u003e\nmc1006.eqiad.wmnet: True\u003cbr \/\u003e\nmc1007.eqiad.wmnet: True\u003cbr \/\u003e\nmc1008.eqiad.wmnet: True\u003cbr \/\u003e\nmc1009.eqiad.wmnet: True\u003cbr \/\u003e\nmc1010.eqiad.wmnet: True\u003cbr \/\u003e\nmc1011.eqiad.wmnet: True\u003cbr \/\u003e\nmc1012.eqiad.wmnet: True\u003cbr \/\u003e\nmc1013.eqiad.wmnet: True\u003cbr \/\u003e\nmc1014.eqiad.wmnet: True\u003cbr \/\u003e\nmc1015.eqiad.wmnet: True\u003cbr \/\u003e\nmc1016.eqiad.wmnet: True\u003cbr \/\u003e\nmc1017.eqiad.wmnet: True\u003cbr \/\u003e\nmc1018.eqiad.wmnet: True\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1985735\" id=\"1985735\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_93\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1985735\" data-sigil=\"has-tooltip\" data-meta=\"0_92\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 1 2016, 9:42 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-01 09:42:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_90\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_91\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eUpdates from me (or better excuses to justify why this work hasn't been completed yet :)\u003c\/p\u003e\n\n\u003cp\u003eI worked with Joe to double check how nutcracker handles redis and memcached nodes going away from its pools and what is the work to be done for the nodes used as lock manager in redis.\u003c\/p\u003e\n\n\u003cp\u003eResults:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003enutcracker does its job removing nodes from the pool after few failures (configured to 3) returning connection errors to the client. This behavior of course trigger a small and limited impact while removing a node, but it gives us the flexibility for this maintenance task.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003col class=\"remarkup-list\" start=\"2\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eredis lock manager will need to be removed from mediawiki-config (together with a subsequent deployment).\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003col class=\"remarkup-list\" start=\"3\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAs Joe already mentioned, each time that a node goes down a small part of the user sessions get deleted with it.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eNext steps for me:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWork with Joe and Dzahn to figure out how to provision a new node with Jessie\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePrepare a plan about rolling out the new OS in a controlled way to limit the impact as much as possible.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eSpecial mention to Joe for his patience up to now :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1993217\" id=\"1993217\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_102\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1993217\" data-sigil=\"has-tooltip\" data-meta=\"0_101\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 3 2016, 9:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-03 09:11:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_99\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_100\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_29\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T123711#1972487\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT123711#1972487\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Dzahn\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Dzahn\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003elist of servers:\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eLock Manager Redis, need a change in operations\/mediawiki-config beforehand\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003emc1001.eqiad.wmnet: True\u003cbr \/\u003e\nmc1002.eqiad.wmnet: True\u003cbr \/\u003e\nmc1003.eqiad.wmnet: True\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eStandard cache nodes\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003emc1004.eqiad.wmnet: True\u003cbr \/\u003e\nmc1005.eqiad.wmnet: True\u003cbr \/\u003e\nmc1006.eqiad.wmnet: True\u003cbr \/\u003e\nmc1007.eqiad.wmnet: True\u003cbr \/\u003e\nmc1008.eqiad.wmnet: True\u003cbr \/\u003e\nmc1009.eqiad.wmnet: True\u003cbr \/\u003e\nmc1010.eqiad.wmnet: True\u003cbr \/\u003e\nmc1011.eqiad.wmnet: True\u003cbr \/\u003e\nmc1012.eqiad.wmnet: True\u003cbr \/\u003e\nmc1013.eqiad.wmnet: True\u003cbr \/\u003e\nmc1014.eqiad.wmnet: True\u003cbr \/\u003e\nmc1015.eqiad.wmnet: True\u003cbr \/\u003e\nmc1016.eqiad.wmnet: True\u003cbr \/\u003e\nmc1017.eqiad.wmnet: True\u003cbr \/\u003e\nmc1018.eqiad.wmnet: True\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_112\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1995925\" id=\"1995925\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003egerritbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1995925\" data-sigil=\"has-tooltip\" data-meta=\"0_110\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 3 2016, 10:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-03 22:50:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_108\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_30\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 268311 had a related patch set uploaded (by Dzahn):\u003cbr \/\u003e\ndhcp: switch mc1004\/1005 to jessie installer\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/268311\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/268311\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1995926\" id=\"1995926\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1995926\" data-sigil=\"has-tooltip\" data-meta=\"0_113\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 3 2016, 10:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-03 22:50:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1995936\" id=\"1995936\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_124\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1995936\" data-sigil=\"has-tooltip\" data-meta=\"0_123\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 3 2016, 10:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-03 22:52:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_121\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_35\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T123711#1985735\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT123711#1985735\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/elukey\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@elukey\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWork with Joe and Dzahn to figure out how to provision a new node with Jessie\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eHere's an example change that switches mc1004 and 1005 , the first 2 regular cache nodes, in the DHCP config to use the jessie installer once they get PXE booted.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/#\/c\/268311\/1\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/#\/c\/268311\/1\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eYou would merge something like this (or amend and do them all at once) and then run puppet on \u003ctt class=\"remarkup-monospaced\"\u003ecarbon\u003c\/tt\u003e, the DHCP\/TFPT\/install server.\u003c\/p\u003e\n\n\u003cp\u003eThen, from a bastion host, connect to the mgmt console of the first server.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003essh root@mc1004.mgmt.eqiad.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eand use the mgmt password (did you get that yet?) to login\u003c\/p\u003e\n\n\u003cp\u003ethen "console com2" should get you a console view., maybe hit Enter a couple times.\u003c\/p\u003e\n\n\u003cp\u003ethen reboot into PXE (depending on your setup just hitting \u003ca href=\"\/F12\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eF12\u003c\/span\u003e\u003c\/a\u003e works or you need to use racadm commands)\u003c\/p\u003e\n\n\u003cp\u003eYou can find details on \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Platform-specific_documentation\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Platform-specific_documentation\u003c\/a\u003e  after checking which hardware these servers are.\u003c\/p\u003e\n\n\u003cp\u003eYou can check what they are on \u003ca href=\"https:\/\/racktables.wikimedia.org\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/racktables.wikimedia.org\u003c\/a\u003e (you probably need to request a login for that)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1998201\" id=\"1998201\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_133\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1998201\" data-sigil=\"has-tooltip\" data-meta=\"0_132\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 4 2016, 4:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-04 16:02:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_130\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_131\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Dzahn\u003c\/span\u003e\u003c\/a\u003e, I wrote this wiki page:\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/User:Elukey\/Ops\/JessieMigration\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/User:Elukey\/Ops\/JessieMigration\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eThe idea would be to do 1004\/5 tomorrow, and then try the wmf-reimage script (asking Ariel\/Joe beforehand because the salt part doesn't work).\u003c\/p\u003e\n\n\u003cp\u003eHow does it sound?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1999516\" id=\"1999516\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1999516\" data-sigil=\"has-tooltip\" data-meta=\"0_141\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 4 2016, 7:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-04 19:59:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_139\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T123711#1998201\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT123711#1998201\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/elukey\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@elukey\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eHow does it sound?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIt sounds good! The wiki page is very detailed and links to exisiting procedures and the plan sounds solid.\u003c\/p\u003e\n\n\u003cp\u003efwiw, we also just went through testing the serial console access.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_152\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2002552\" id=\"2002552\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_151\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2002552\" data-sigil=\"has-tooltip\" data-meta=\"0_150\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2016, 4:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-05 16:51:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_148\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAfter a chat with Joe with decided to postpone the start of the work to Monday morning to avoid big changes on a Friday (especially done by me).\u003c\/p\u003e\n\n\u003cp\u003eI'll start Monday morning with m1004\/m1005\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2007885\" id=\"2007885\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_160\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2007885\" data-sigil=\"has-tooltip\" data-meta=\"0_159\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2016, 3:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-08 15:04:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_157\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 268311 merged by Elukey:\u003cbr \/\u003e\ndhcp: switch mc1004\/1005 to jessie installer\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/268311\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/268311\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_164\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2008606\" id=\"2008606\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_163\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eori\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2008606\" data-sigil=\"has-tooltip\" data-meta=\"0_162\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2016, 6:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-08 18:13:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_173\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2008944\" id=\"2008944\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_172\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003eelukey\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2008944\" data-sigil=\"has-tooltip\" data-meta=\"0_171\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 8 2016, 6:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-02-08 18:42:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_169\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_170\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eUpdate about today:\u003c\/p\u003e\n\n\u003cp\u003eI followed \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/User:Elukey\/Ops\/JessieMigration\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/User:Elukey\/Ops\/JessieMigration\u003c\/a\u003e with mc1004 starting with:\u003c\/p\u003e\n\n\u003cp\u003esudo service redis-server stop (this one is wrong, because it is needed redis-instance-tcp_6379 instead)\u003cbr \/\u003e\nsudo service memcached stop\u003c\/p\u003e\n\n\u003cp\u003eI checked \u003ca href=\"https:\/\/logstash.wikimedia.org\/#\/dashboard\/elasticsearch\/memcached\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/logstash.wikimedia.org\/#\/dashboard\/elasticsearch\/memcached\u003c\/a\u003e and a lot errors were logged. Top ones:\u003c\/p\u003e\n\n\u003cp\u003eMemcached error for key "WANCache:t:dewiki:messages:en" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t1541\t \u003cbr \/\u003e\nMemcached error for key "WANCache:v:commonswiki:revisiontext:textid:141778461" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t1349\t \u003cbr \/\u003e\nMemcached error for key "global:lag-times:db1033" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t1309\t \u003cbr \/\u003e\nMemcached error for key "WANCache:v:enwiki:revisiontext:textid:607643685" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t1247\t \u003cbr \/\u003e\nMemcached error for key "frwiki:preprocess-hash:736905796af92b2e8e930cebe6c0b31b:0" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t676\t \u003cbr \/\u003e\nMemcached error for key "WANCache:t:trwiki:gadgets-definition:9:2" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t524\t \u003cbr \/\u003e\nMemcached error for key "WANCache:t:fawiki:gadgets-definition:9:2" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t444\t \u003cbr \/\u003e\nMemcached error for key "WANCache:v:enwiktionary:revisiontext:textid:36970227" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t366\t \u003cbr \/\u003e\nMemcached error for key "WANCache:v:commonswiki:revisiontext:textid:105720496" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t289\t \u003cbr \/\u003e\nMemcached error for key "WANCache:v:commonswiki:revisiontext:textid:178315826" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t265\t \u003cbr \/\u003e\nMemcached error for key "WANCache:t:ruwikisource:messages:ru" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t209\t \u003cbr \/\u003e\nMemcached error for key "WANCache:t:commonswiki:messages:fr" on server "\/var\/run\/nutcracker\/nutcracker.sock:0": SERVER ERROR\t197\u003c\/p\u003e\n\n\u003cp\u003eI then started to restart nutcracker in an attempt to fix the issue, but didn't manage to reduce the impact (even after an update of all the nutcracker instances running in G@cluster:appserver and G@site:eqiad servers with salt).\u003c\/p\u003e\n\n\u003cp\u003eFrom my tests nutcracker should have avoided this removing the host from the pool of servers right after some errors thrown.\u003c\/p\u003e\n\n\u003cp\u003eIt was not related to a few hosts: (taken from logstash's top hosts panel for the duration of the issue)\u003c\/p\u003e\n\n\u003cp\u003emw1024 (244)  mw1168 (217)  mw1057 (200)  mw1109 (185)  mw1239 (184)  mw1070 (183)  mw1245 (182)  mw1238 (178)  mw1021 (176)  mw1243 (173)  mw1216 (170)  mw1255 (168)  mw1059 (168)  mw1210 (165)  mw1055 (165)  mw1048 (163)  mw1014 (162)  mw1042 (161)  mw1209 (160)  mw1100 (160)\u003c\/p\u003e\n\n\u003cp\u003eNext step to unblock this rollout would be to remove the mc hosts from the nutcracker list before proceeding, but also it would be great if we could figure out why nutcracker didn't behave as expected.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd","contextPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu","contextPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-FILE-sh3q7wfft7sdx7z5edtf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd","contextPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu","contextPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd","contextPHID":"PHID-TASK-vxby2wgflnxdrepehyfq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6p4p4vbpzsydxbxewmdj"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-mzylqwmroro3a2g"},{"phid":"PHID-XACT-TASK-7lx2knyyhllczju"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-fht6dmyi6uef7le"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-vlglrfzybg3im4q"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-jhzzo3uo7e3tce2"},{"phid":"PHID-XACT-TASK-xika4qqwpl6p2qy"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-gghye2qa64q6xl6"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-rtcssulrzit2fbv"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-xbbyorstxgb4ny5"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-b4qm2wgut4h7g63"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-j3khw3hk36pqs74"},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"phid":"PHID-XACT-TASK-s4upbvx7cyfur3u"},{"tip":"Via Web"},[],[],[],[],[],{"phid":"PHID-XACT-TASK-aegpxoy4c4c3zyq","anchor":"1936342"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-opc2oztktzoyrax","anchor":"1962208"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mzylqwmroro3a2g\/","ref":"T123711#1963069"},[],{"anchor":"1963069"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_57\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_58\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mzylqwmroro3a2g\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_59\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_60\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mzylqwmroro3a2g","anchor":"1963069"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7lx2knyyhllczju\/","ref":"T123711#1965105"},[],{"anchor":"1965105"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_66\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_67\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7lx2knyyhllczju\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_68\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_69\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-7lx2knyyhllczju","anchor":"1965105"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fht6dmyi6uef7le\/","ref":"T123711#1972487"},[],{"anchor":"1972487"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_77\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_78\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fht6dmyi6uef7le\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_79\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_80\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fht6dmyi6uef7le","anchor":"1972487"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vlglrfzybg3im4q\/","ref":"T123711#1985735"},[],{"anchor":"1985735"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_86\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_87\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vlglrfzybg3im4q\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_88\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_89\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vlglrfzybg3im4q","anchor":"1985735"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jhzzo3uo7e3tce2\/","ref":"T123711#1993217"},[],{"anchor":"1993217"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_95\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jhzzo3uo7e3tce2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_97\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_98\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jhzzo3uo7e3tce2","anchor":"1993217"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xika4qqwpl6p2qy\/","ref":"T123711#1995925"},[],{"anchor":"1995925"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_104\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xika4qqwpl6p2qy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_106\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-xika4qqwpl6p2qy","anchor":"1995925"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-h2dnayox3q4z76g","anchor":"1995926"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gghye2qa64q6xl6\/","ref":"T123711#1995936"},[],{"anchor":"1995936"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_116\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_117\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gghye2qa64q6xl6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_118\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_119\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-gghye2qa64q6xl6\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gghye2qa64q6xl6","anchor":"1995936"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rtcssulrzit2fbv\/","ref":"T123711#1998201"},[],{"anchor":"1998201"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_126\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rtcssulrzit2fbv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_128\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rtcssulrzit2fbv","anchor":"1998201"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xbbyorstxgb4ny5\/","ref":"T123711#1999516"},[],{"anchor":"1999516"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_135\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xbbyorstxgb4ny5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xbbyorstxgb4ny5","anchor":"1999516"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-b4qm2wgut4h7g63\/","ref":"T123711#2002552"},[],{"anchor":"2002552"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-b4qm2wgut4h7g63\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_146\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-b4qm2wgut4h7g63","anchor":"2002552"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-j3khw3hk36pqs74\/","ref":"T123711#2007885"},[],{"anchor":"2007885"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_153\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_154\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j3khw3hk36pqs74\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_155\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-j3khw3hk36pqs74","anchor":"2007885"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4m63nm5gmjfydk5","anchor":"2008606"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-s4upbvx7cyfur3u\/","ref":"T123711#2008944"},[],{"anchor":"2008944"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_165\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-s4upbvx7cyfur3u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_167\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-s4upbvx7cyfur3u","anchor":"2008944"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}