for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/uikgw65kdhdz2jnce3ik\/PHID-FILE-cybxba6527abtnb3af5x\/alphanumeric_lato-white_R.png-_7b4b49-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/rtimport\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"930644\" id=\"930644\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/rtimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e rtimport\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003e\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930644\" data-sigil=\"has-tooltip\" data-meta=\"0_87\"\u003e\u003cspan class=\"screen-only\"\u003eDec 18 2014, 2:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-12-18 02:19:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_89\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/rtimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e rtimport\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/ops-core\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003eops-core\u003c\/a\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/rtimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e rtimport\u003c\/a\u003e set Reference to rt8839.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"930651\" id=\"930651\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_93\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003eDzahn\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930651\" data-sigil=\"has-tooltip\" data-meta=\"0_92\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2014, 7:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-10 19:54:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_103\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_104\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"930659\" id=\"930659\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_102\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930659\" data-sigil=\"has-tooltip\" data-meta=\"0_101\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2014, 8:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-10 20:01:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_99\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_100\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_16\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp class=\"remarkup-literal\"\u003eOn Mon Nov 10 19:54:36 2014, dzahn wrote:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eas mentioned in ops meeting today and requested by Giuseppe:\u003c\/p\u003e\n\n\u003cp\u003ereinstall OCG servers with a new partitioning scheme, use LVM, leave\u003cbr \/\u003e\nsome free\u003cbr \/\u003e\nextents, one by one\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp class=\"remarkup-literal\"\u003eMy suggestions:\n\u003cbr \/\u003e- make \/, \/var\/log and \/srv as separate partitions, make them based on actual\n\u003cbr \/\u003edisk usage (but .\/srv is going to be larger in the future, check with cscott)\n\u003cbr \/\u003e- use LVM and leave at least a 20% of unallocated space\n\u003cbr \/\u003e- maybe keep the software RAID beneath that - I don't really see a reason for\n\u003cbr \/\u003ehaving it, but think it through.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_114\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/uikgw65kdhdz2jnce3ik\/PHID-FILE-cybxba6527abtnb3af5x\/alphanumeric_lato-white_R.png-_7b4b49-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/rtimport\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"930672\" id=\"930672\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_113\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/rtimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e rtimport\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930672\" data-sigil=\"has-tooltip\" data-meta=\"0_112\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2014, 8:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-10 20:01:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_110\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_18\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cem\u003eStatus changed from 'new' to 'open' by RT_System\u003c\/em\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"930696\" id=\"930696\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930696\" data-sigil=\"has-tooltip\" data-meta=\"0_121\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2014, 8:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-10 20:09:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_119\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_20\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIssue taken by \u003cstrong\u003edzahn\u003c\/strong\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_132\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"930708\" id=\"930708\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_131\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930708\" data-sigil=\"has-tooltip\" data-meta=\"0_130\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2014, 12:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-11 00:16:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_128\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_22\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp class=\"remarkup-literal\"\u003e14:45 <cscott> i don't have a strong opinion on software raid. all of the data\n\u003cbr \/\u003eon the machine is just cache, it can be easily reconstructed if there's a disk\n\u003cbr \/\u003efailure.\n\u003cbr \/\u003e14:49 <cscott> if i had a choice between software raid and a lot more disk\n\u003cbr \/\u003espace, i'd like to try a lot more disk space. :)\n\u003cbr \/\u003e---\n\u003cbr \/\u003e14:42 <cscott> ok, deploy finished. i just increased the cache lifetime for ocg\n\u003cbr \/\u003efrom 2 days to 4, since we're currently using only 15% of our available cache\n\u003cbr \/\u003espace on\n\u003cbr \/\u003e\/srv.\n\u003cbr \/\u003e14:42 <cscott> so it might be good to wait at least 2 days to let that settle\n\u003cbr \/\u003edown first\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"930728\" id=\"930728\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930728\" data-sigil=\"has-tooltip\" data-meta=\"0_139\"\u003e\u003cspan class=\"screen-only\"\u003eDec 3 2014, 1:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-12-03 01:56:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_137\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_24\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp class=\"remarkup-literal\"\u003erequirement for this task is:\n\u003cbr \/\u003ehttps:\/\/gerrit.wikimedia.org\/r\/#\/c\/177120\/\n\u003cbr \/\u003ea script to allow taking one of the servers at a time out of rotation to\n\u003cbr \/\u003ereinstall\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"930742\" id=\"930742\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#930742\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"screen-only\"\u003eDec 3 2014, 1:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-12-03 01:56:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_146\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_26\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cem\u003eStatus changed from 'open' to 'stalled' by dzahn\u003c\/em\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"980770\" id=\"980770\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003eDzahn\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#980770\" data-sigil=\"has-tooltip\" data-meta=\"0_151\"\u003e\u003cspan class=\"screen-only\"\u003eJan 15 2015, 10:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-01-15 22:51:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_157\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"998558\" id=\"998558\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eDzahn\u003c\/a\u003e edited projects, added \u003ca href=\"\/tag\/acl_sre-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eacl*sre-team\u003c\/a\u003e; removed \u003ca href=\"\/tag\/ops-core\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eops-core\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#998558\" data-sigil=\"has-tooltip\" data-meta=\"0_154\"\u003e\u003cspan class=\"screen-only\"\u003eJan 28 2015, 5:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-01-28 17:45:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eDzahn\u003c\/a\u003e set Security to None.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_166\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1002853\" id=\"1002853\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_165\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eDzahn\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/cscott\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003ecscott\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1002853\" data-sigil=\"has-tooltip\" data-meta=\"0_164\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2015, 12:20 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-01-30 00:20:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_162\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_163\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_34\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/cscott\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_1\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cscott\u003c\/span\u003e\u003c\/a\u003e How's the status of those patches we needed to take one of the hosts out of rotation?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_170\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2nco5z2tyvhf6dfxheoj\/PHID-FILE-pacxilbspughn725akdz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mark\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1008046\" id=\"1008046\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mark\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003emark\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/blocked-on-parsoid\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eBlocked-on-Parsoid\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1008046\" data-sigil=\"has-tooltip\" data-meta=\"0_167\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 2 2015, 11:38 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-02 11:38:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mark\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003emark\u003c\/a\u003e subscribed.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/z67mikd4lwdetm774u5v\/PHID-FILE-wzypambajaxwezctdezx\/profile-livejournal.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cscott\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1029110\" id=\"1029110\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_178\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cscott\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003ecscott\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1029110\" data-sigil=\"has-tooltip\" data-meta=\"0_177\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 10 2015, 9:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-10 21:20:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_175\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_176\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe scripts were deployed on \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/OCG\/Deployments#2014-12-03\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003e2014-12-03\u003c\/a\u003e and the last cache entries without associated host data should have expired a week or so after that.\u003c\/p\u003e\n\n\u003cp\u003eI added some documentation on the process at \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/OCG#Decommissioning_a_host\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/OCG#Decommissioning_a_host\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eI tested this locally, but we should try to ensure that I'm online when this process is done the first time so I can verify that the documented steps actually work correctly.\u003c\/p\u003e\n\n\u003cp\u003eAlso, I'm not entirely sure how the round-robin DNS setup works, so the linked documentation could probably be expanded a bit with more concrete details on that part.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1032831\" id=\"1032831\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_188\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1032831\" data-sigil=\"has-tooltip\" data-meta=\"0_187\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 11 2015, 10:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-11 22:23:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_185\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks for the docs! I will amend it, the round-robin setup works via \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Pybal\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003epybal\u003c\/a\u003e. We connect to config-master.eqiad.wmnet and make a pybal config change in \/srv\/pybal-config and git commit it locally.\u003c\/p\u003e\n\n\u003cp\u003ei just disabled ocg1003 and the config looks like this:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eocg:{ 'host': 'ocg1001.eqiad.wmnet', 'weight': 10, 'enabled': True }\nocg:{ 'host': 'ocg1002.eqiad.wmnet', 'weight': 10, 'enabled': True }\nocg:{ 'host': 'ocg1003.eqiad.wmnet', 'weight': 10, 'enabled': False }\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ewill wait a while now and watch \u003ca href=\"http:\/\/ganglia.wikimedia.org\/latest\/?c=PDF%20servers%20eqiad&m=cpu_report&r=hour&s=descending&hc=4&mc=2\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eganglia\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_192\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1032832\" id=\"1032832\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_191\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eDzahn\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/blocked-on-parsoid\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003eBlocked-on-Parsoid\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1032832\" data-sigil=\"has-tooltip\" data-meta=\"0_190\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 11 2015, 10:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-11 22:24:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_201\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1032851\" id=\"1032851\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_200\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eDzahn\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003ebd808\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1032851\" data-sigil=\"has-tooltip\" data-meta=\"0_199\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 11 2015, 10:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-11 22:27:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_197\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_198\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bd808\u003c\/span\u003e\u003c\/a\u003e added you "fyi", because cscott said "check with bd808, he's having some trouble with logstash and ocg. i don't think this should affect him, but it's worth double-checking"\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_212\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_210\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_211\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1033032\" id=\"1033032\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1033032\" data-sigil=\"has-tooltip\" data-meta=\"0_208\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 12 2015, 12:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-12 00:11:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_206\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_207\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T84723#1032851\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT84723#1032851\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Dzahn\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Dzahn\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bd808\u003c\/span\u003e\u003c\/a\u003e added you "fyi", because cscott said "check with bd808, he's having some trouble with logstash and ocg. i don't think this should affect him, but it's worth double-checking"\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eNo blocker from me. OCG is a chatty chatty logger and Logstash is dying from Java OOM\/GC weight but we have other tickets open to work on that.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_221\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1033450\" id=\"1033450\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1033450\" data-sigil=\"has-tooltip\" data-meta=\"0_219\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 12 2015, 3:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-12 03:09:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_217\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003edoes this look like it just kept going even though i disabled it in pybal?\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"http:\/\/ganglia.wikimedia.org\/latest\/graph_all_periods.php?c=PDF%20servers%20eqiad&h=ocg1003.eqiad.wmnet&r=hour&z=default&jr=&js=&st=1423710463&v=695595&m=ocg_job_status_queue&vl=messages&ti=ocg_job_status_queue&z=large\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/ganglia.wikimedia.org\/latest\/graph_all_periods.php?c=PDF%20servers%20eqiad&h=ocg1003.eqiad.wmnet&r=hour&z=default&jr=&js=&st=1423710463&v=695595&m=ocg_job_status_queue&vl=messages&ti=ocg_job_status_queue&z=large\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_231\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1035760\" id=\"1035760\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_230\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1035760\" data-sigil=\"has-tooltip\" data-meta=\"0_229\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 13 2015, 12:42 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-13 00:42:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_227\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_52\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e16:25 < bblack> mutante: the graphs do look similar between 100x like nothing changed for 1003. ..\u003c\/p\u003e\n\n\u003cp\u003e16:26 < mutante> bblack: i went to config-master.eqiad, then \/srv\/pybal-config\/ and committed\u003c\/p\u003e\n\n\u003cp\u003e16:27 < bblack> 2015-02-13 00:27:07.885550 [ocg_8000 ProxyFetch] ocg1002.eqiad.wmnet (enabled\/up\/pooled): Fetch successful, 0.021 s\u003cbr \/\u003e\n16:27 < bblack> 2015-02-13 00:27:08.690954 [ocg_8000 ProxyFetch] ocg1003.eqiad.wmnet (disabled\/up\/not pooled): Fetch successful, 0.007 s\u003cbr \/\u003e\n16:27 < bblack> 2015-02-13 00:27:11.618436 [ocg_8000 ProxyFetch] ocg1001.eqiad.wmnet (enabled\/up\/pooled): Fetch successful, 0.006 s\u003c\/p\u003e\n\n\u003cp\u003e16:27 < mutante> bblack: ocg1003 is supposed to be disabled so that i can reinstall it. the docs say to turn it off and wait for the jobs to get finished. but i'm \u003cbr \/\u003e\nconfused because nothing happened\u003c\/p\u003e\n\n\u003cp\u003e16:30 < bblack> yeah there's no pybal bug either, the raw ipvs tables have just ocg100[12] IPs:\u003cbr \/\u003e\n16:30 < bblack> TCP  10.2.2.31:8000 wrr -> 10.64.32.151:8000            Route   10     0          564 -> 10.64.48.42:8000             Route   10     0          565\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_235\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1068972\" id=\"1068972\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003eDzahn\u003c\/a\u003e mentioned this in \u003ca href=\"\/T90839\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eT90839: pybal issue?\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1068972\" data-sigil=\"has-tooltip\" data-meta=\"0_232\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 26 2015, 4:01 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-02-26 04:01:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003eDzahn\u003c\/a\u003e added a subtask: \u003ca href=\"\/T90839\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eT90839: pybal issue?\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_244\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1137632\" id=\"1137632\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1137632\" data-sigil=\"has-tooltip\" data-meta=\"0_242\"\u003e\u003cspan class=\"screen-only\"\u003eMar 21 2015, 12:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-21 00:49:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_240\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_58\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003esince the pybal issue has been fixed by a restart of it and works fine for other servers again this means i still don't know how to properly depool an ocg server :\/\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_248\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1137634\" id=\"1137634\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock phui-timeline-icon\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003eDzahn\u003c\/a\u003e changed the visibility from \"\u003ca href=\"\/tag\/wmf-nda\/\"\u003eWMF-NDA\u003c\/a\u003e (Project)\" to \"Public (No Login Required)\".\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1137634\" data-sigil=\"has-tooltip\" data-meta=\"0_245\"\u003e\u003cspan class=\"screen-only\"\u003eMar 21 2015, 12:50 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-21 00:50:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock phui-timeline-icon\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003eDzahn\u003c\/a\u003e changed the edit policy from \"\u003ca href=\"\/tag\/wmf-nda\/\"\u003eWMF-NDA\u003c\/a\u003e (Project)\" to \"All Users\".\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_257\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1137644\" id=\"1137644\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1137644\" data-sigil=\"has-tooltip\" data-meta=\"0_255\"\u003e\u003cspan class=\"screen-only\"\u003eMar 21 2015, 12:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-21 00:56:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_253\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_254\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e{ 'host': 'ocg1003.eqiad.wmnet', 'weight': 10, 'enabled': False }\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"http:\/\/ganglia.wikimedia.org\/latest\/graph.php?r=week&z=xlarge&h=ocg1003.eqiad.wmnet&m=cpu_report&s=descending&mc=2&g=network_report&c=PDF+servers+eqiad\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/ganglia.wikimedia.org\/latest\/graph.php?r=week&z=xlarge&h=ocg1003.eqiad.wmnet&m=cpu_report&s=descending&mc=2&g=network_report&c=PDF+servers+eqiad\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_260\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1137650\" id=\"1137650\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_259\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003eDzahn\u003c\/a\u003e closed subtask \u003ca href=\"\/T90839\" class=\"phui-handle handle-status-closed\"\u003eT90839: pybal issue?\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1137650\" data-sigil=\"has-tooltip\" data-meta=\"0_258\"\u003e\u003cspan class=\"screen-only\"\u003eMar 21 2015, 12:57 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-21 00:57:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_269\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1137679\" id=\"1137679\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_268\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1137679\" data-sigil=\"has-tooltip\" data-meta=\"0_267\"\u003e\u003cspan class=\"screen-only\"\u003eMar 21 2015, 1:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-21 01:19:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_265\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_266\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ewhile you would think ocg1003 stops getting jobs, it looks actually like this:\u003c\/p\u003e\n\n\u003cp\u003eocg1002 and ocg1003 are both doing a lot of stuff (tail -f \/srv\/deployment\/ocg\/log\/ocg.log), while ocg1001 is the one not doing anything. the log here is actually empty. why is that?\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e0 -rw-r--r-- 1 syslog adm       0 Mar 17 06:25 ocg.log\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ethe command mentioned on \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/OCG#Decommissioning_a_host\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/OCG#Decommissioning_a_host\u003c\/a\u003e to depool a host is "command not found". where do you run that?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-black\" data-sigil=\"transaction anchor-container\" data-meta=\"0_278\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/w7vkzz5perxsvlcjueb5\/PHID-FILE-jlr7ilkp27awws5w7x5u\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1158963\" id=\"1158963\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-black\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_277\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003eDzahn\u003c\/a\u003e removed \u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\"\u003eDzahn\u003c\/a\u003e as the assignee of this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1158963\" data-sigil=\"has-tooltip\" data-meta=\"0_276\"\u003e\u003cspan class=\"screen-only\"\u003eMar 28 2015, 12:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-28 00:22:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_274\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_275\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ecould somebody help me here and take a look?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_282\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1164054\" id=\"1164054\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/services\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eServices\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1164054\" data-sigil=\"has-tooltip\" data-meta=\"0_279\"\u003e\u003cspan class=\"screen-only\"\u003eMar 30 2015, 7:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-03-30 19:05:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e subscribed.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_291\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1170722\" id=\"1170722\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_290\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1170722\" data-sigil=\"has-tooltip\" data-meta=\"0_289\"\u003e\u003cspan class=\"screen-only\"\u003eApr 1 2015, 4:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-04-01 16:54:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_287\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_288\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/cscott\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cscott\u003c\/span\u003e\u003c\/a\u003e, is it possible that the network transfer seen by \u003ca href=\"\/p\/Dzahn\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Dzahn\u003c\/span\u003e\u003c\/a\u003e is caused by the depooled node still processing jobs from the queue?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_300\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/z67mikd4lwdetm774u5v\/PHID-FILE-wzypambajaxwezctdezx\/profile-livejournal.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cscott\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1172797\" id=\"1172797\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_299\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cscott\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003ecscott\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1172797\" data-sigil=\"has-tooltip\" data-meta=\"0_298\"\u003e\u003cspan class=\"screen-only\"\u003eApr 2 2015, 2:10 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-04-02 02:10:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_296\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_297\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_74\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eUpdated the wikipage to make the directory and host clear for the "decommissioning a host" script.\u003c\/p\u003e\n\n\u003cp\u003eIf that script hasn't been run, then the 7-day (I think, I forget the exact duration) redis cache will still point to entries on the decommissioned host, so it will get HTTP traffic to fetch them.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e: it's possible that the depooled node will still request jobs from the queue.  I'm a bit fuzzy on that part of the code, I could re-read it to be sure.  My idea was that you'd take down the service (so that it didn't request new jobs) and then immediately run the "decommissioning a host script" (so that there were no cache entries pointing to the down host).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_309\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1172917\" id=\"1172917\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1172917\" data-sigil=\"has-tooltip\" data-meta=\"0_307\"\u003e\u003cspan class=\"screen-only\"\u003eApr 2 2015, 4:07 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-04-02 04:07:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_305\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_306\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_76\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/cscott\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cscott\u003c\/span\u003e\u003c\/a\u003e, could you have a look on ocg1003 to see what's going on?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-grey\" data-sigil=\"transaction anchor-container\" data-meta=\"0_313\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6n6ztqf7kwlq2m3tns3r\/PHID-FILE-sjs4kdocejuiiypnxqow\/profile-lake.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/chasemp\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1327152\" id=\"1327152\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-grey\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-down phui-timeline-icon\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/chasemp\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e chasemp\u003c\/a\u003e lowered the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1327152\" data-sigil=\"has-tooltip\" data-meta=\"0_310\"\u003e\u003cspan class=\"screen-only\"\u003eJun 1 2015, 9:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-06-01 21:11:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/chasemp\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e chasemp\u003c\/a\u003e subscribed.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_324\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2k47tvkka236nmpz22pc\/PHID-FILE-h42cqpdrg4yr7egelavj\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Aklapper\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/2\/\" data-sigil=\"has-tooltip\" data-meta=\"0_322\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-bug\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1369719\" id=\"1369719\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eAklapper\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1369719\" data-sigil=\"has-tooltip\" data-meta=\"0_320\"\u003e\u003cspan class=\"screen-only\"\u003eJun 16 2015, 2:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-06-16 14:37:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_318\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_80\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eStill an issue?\u003cbr \/\u003e\nIf so, \u003ca href=\"\/p\/cscott\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cscott\u003c\/span\u003e\u003c\/a\u003e, could you reply to the last comment?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_335\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_333\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1441330\" id=\"1441330\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eJoe\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1441330\" data-sigil=\"has-tooltip\" data-meta=\"0_331\"\u003e\u003cspan class=\"screen-only\"\u003eJul 9 2015, 2:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-09 14:20:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_329\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eocg1003 is offline since months, and we really ought to re-commission it.\u003c\/p\u003e\n\n\u003cp\u003eI see the server was never reinstalled, in fact it's just grabbing jobs from the jobqueue and not getting any traffic from the load balancer.\u003c\/p\u003e\n\n\u003cp\u003eWe need to reinstall it ASAP.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_338\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1441331\" id=\"1441331\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Matanya\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003eMatanya\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/815622\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1441331\" data-sigil=\"has-tooltip\" data-meta=\"0_336\"\u003e\u003cspan class=\"screen-only\"\u003eJul 9 2015, 2:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-09 14:20:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_341\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1441332\" id=\"1441332\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-exclamation-circle phui-timeline-icon\" data-meta=\"0_340\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eJoe\u003c\/a\u003e changed the task status from \u003cspan class=\"phui-timeline-value\"\u003eStalled\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1441332\" data-sigil=\"has-tooltip\" data-meta=\"0_339\"\u003e\u003cspan class=\"screen-only\"\u003eJul 9 2015, 2:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-09 14:20:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-j3rt67f2xk5rpkgcegnj"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats","contextPHID":"PHID-TASK-sw64kkxrhih7slbbcady"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j3rt67f2xk5rpkgcegnj"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-jz52p6k5r7mbubnjugwo"}},{"hovercardSpec":{"objectPHID":"PHID-USER-j3rt67f2xk5rpkgcegnj"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-rc33oh2xpxkxlci"},{"hovercardSpec":{"objectPHID":"PHID-USER-j3rt67f2xk5rpkgcegnj"}},{"phid":"PHID-XACT-TASK-nybcm3l6izbfuy7"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-bcqyvnru7pooknu"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-ousgw7jgnlnnzi7"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-7eh6pfolfg7x4f6"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-wmqokzwrwo4nna4"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-fqb3bhereyljcqrsbju7"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-jz52p6k5r7mbubnjugwo"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats"}},{"phid":"PHID-XACT-TASK-a6sbfaz5k3ojot7"},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ttd3f7i5qibnfgbs6cxl"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats"}},{"phid":"PHID-XACT-TASK-6psi2jgttml3rnu"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-dymz7v4rmdvj5la"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ttd3f7i5qibnfgbs6cxl"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-yjqme2gxmkavwlo"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-bgp7cbfzsi4kc22"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-jmbm6ofr6sfisct"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-ab5sh3mbcjbnfl2"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3hhfu3bbvtda6tfqvrld"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3hhfu3bbvtda6tfqvrld"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-2qaiqw4nqto3z2l"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-n4ydu4wanimw3nd"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-vcfmc52oezfglbo"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-5yulwqljvprgawl"},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-6gl4ltotgleuetlmf2ye"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"phid":"PHID-XACT-TASK-rmk7kazeavkr4tn"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m2ezqyeb4uz67zq6bats"}},{"phid":"PHID-XACT-TASK-t7tlz7o4ayshkrr"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-txg2vyul37uliyd"},{"hovercardSpec":{"objectPHID":"PHID-USER-3neel27i7dyu62jbbx2l"}},{"hovercardSpec":{"objectPHID":"PHID-USER-3neel27i7dyu62jbbx2l"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"phid":"PHID-XACT-TASK-4dj5vz2j6sx4oyw"},{"phid":"PHID-XACT-TASK-u2gijfru764fegz"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-u7udgblfyop6qd5wxot6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"tip":"Via Conduit"},[],[],[],{"phid":"PHID-XACT-TASK-xjzvp3y2mn7zzlt","anchor":"930644"},{"tip":"Via Old World"},[],{"phid":"PHID-XACT-TASK-ab2f0820f021a80","anchor":"930651"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rc33oh2xpxkxlci\/","ref":"T84723#930659"},[],{"anchor":"930659"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_95\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rc33oh2xpxkxlci\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_97\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_98\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rc33oh2xpxkxlci","anchor":"930659"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nybcm3l6izbfuy7\/","ref":"T84723#930672"},[],{"anchor":"930672"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_106\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nybcm3l6izbfuy7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_108\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-nybcm3l6izbfuy7","anchor":"930672"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bcqyvnru7pooknu\/","ref":"T84723#930696"},[],{"anchor":"930696"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_115\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bcqyvnru7pooknu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bcqyvnru7pooknu","anchor":"930696"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ousgw7jgnlnnzi7\/","ref":"T84723#930708"},[],{"anchor":"930708"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_124\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ousgw7jgnlnnzi7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_126\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ousgw7jgnlnnzi7","anchor":"930708"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7eh6pfolfg7x4f6\/","ref":"T84723#930728"},[],{"anchor":"930728"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_133\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7eh6pfolfg7x4f6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_135\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7eh6pfolfg7x4f6","anchor":"930728"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wmqokzwrwo4nna4\/","ref":"T84723#930742"},[],{"anchor":"930742"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_142\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wmqokzwrwo4nna4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wmqokzwrwo4nna4","anchor":"930742"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ynlht5z5auod26h","anchor":"980770"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-r2kkkt32qoup55a","anchor":"998558"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-a6sbfaz5k3ojot7\/","ref":"T84723#1002853"},[],{"anchor":"1002853"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_158\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-a6sbfaz5k3ojot7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_160\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dxbqnf2fbszphip","anchor":"1002853"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-svox5bp6ufkaic5","anchor":"1008046"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6psi2jgttml3rnu\/","ref":"T84723#1029110"},[],{"anchor":"1029110"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_171\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_172\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6psi2jgttml3rnu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_173\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_174\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6psi2jgttml3rnu","anchor":"1029110"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dymz7v4rmdvj5la\/","ref":"T84723#1032831"},[],{"anchor":"1032831"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_180\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_181\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dymz7v4rmdvj5la\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_182\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_183\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-dymz7v4rmdvj5la\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dymz7v4rmdvj5la","anchor":"1032831"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-l4mnwrelh5fkhgt","anchor":"1032832"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yjqme2gxmkavwlo\/","ref":"T84723#1032851"},[],{"anchor":"1032851"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_193\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_194\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yjqme2gxmkavwlo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_195\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_196\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-t2rlafsufcmurtd","anchor":"1032851"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bgp7cbfzsi4kc22\/","ref":"T84723#1033032"},[],{"anchor":"1033032"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_202\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bgp7cbfzsi4kc22\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_204\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_205\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-bgp7cbfzsi4kc22","anchor":"1033032"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jmbm6ofr6sfisct\/","ref":"T84723#1033450"},[],{"anchor":"1033450"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_213\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_214\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jmbm6ofr6sfisct\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_215\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jmbm6ofr6sfisct","anchor":"1033450"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ab5sh3mbcjbnfl2\/","ref":"T84723#1035760"},[],{"anchor":"1035760"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_222\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ab5sh3mbcjbnfl2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_224\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-ab5sh3mbcjbnfl2\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_226\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ab5sh3mbcjbnfl2","anchor":"1035760"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-hu5tuovmnkvcxa2","anchor":"1068972"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2qaiqw4nqto3z2l\/","ref":"T84723#1137632"},[],{"anchor":"1137632"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_236\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2qaiqw4nqto3z2l\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_238\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2qaiqw4nqto3z2l","anchor":"1137632"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-rr65xjdqk4madge","anchor":"1137634"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-n4ydu4wanimw3nd\/","ref":"T84723#1137644"},[],{"anchor":"1137644"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_249\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_250\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-n4ydu4wanimw3nd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_251\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_252\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-n4ydu4wanimw3nd","anchor":"1137644"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dv46aspvotm5sdo","anchor":"1137650"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vcfmc52oezfglbo\/","ref":"T84723#1137679"},[],{"anchor":"1137679"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_261\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vcfmc52oezfglbo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_263\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vcfmc52oezfglbo","anchor":"1137679"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5yulwqljvprgawl\/","ref":"T84723#1158963"},[],{"anchor":"1158963"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_270\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5yulwqljvprgawl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_272\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hyvhx4wabll7cyg","anchor":"1158963"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-vsurd2oazr2y4ft","anchor":"1164054"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rmk7kazeavkr4tn\/","ref":"T84723#1170722"},[],{"anchor":"1170722"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_283\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_284\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rmk7kazeavkr4tn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_285\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_286\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rmk7kazeavkr4tn","anchor":"1170722"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-t7tlz7o4ayshkrr\/","ref":"T84723#1172797"},[],{"anchor":"1172797"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_292\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t7tlz7o4ayshkrr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_294\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-t7tlz7o4ayshkrr","anchor":"1172797"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-txg2vyul37uliyd\/","ref":"T84723#1172917"},[],{"anchor":"1172917"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_301\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_302\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-txg2vyul37uliyd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_303\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_304\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-txg2vyul37uliyd","anchor":"1172917"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-fk3zlm3y6gy7moa","anchor":"1327152"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4dj5vz2j6sx4oyw\/","ref":"T84723#1369719"},[],{"anchor":"1369719"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_314\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_315\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4dj5vz2j6sx4oyw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_316\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Bugwrangler","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-4dj5vz2j6sx4oyw","anchor":"1369719"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u2gijfru764fegz\/","ref":"T84723#1441330"},[],{"anchor":"1441330"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_325\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u2gijfru764fegz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-u2gijfru764fegz","anchor":"1441330"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-ixehrhundfwdssn","anchor":"1441331"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jmont4eaczpkspl","anchor":"1441332"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/602387e0\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}