for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ovytmbleapqqbp2q34as\/PHID-FILE-ukx4a3vvpumxtlt3rof2\/profile-Screen_Shot_2014-11-24_at_10.24.10_AM.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bzimport\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"747905\" id=\"747905\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_44\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bzimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e bzimport\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003e\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747905\" data-sigil=\"has-tooltip\" data-meta=\"0_43\"\u003e\u003cspan class=\"screen-only\"\u003eNov 22 2014, 3:44 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-22 03:44:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_45\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bzimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e bzimport\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/continuous-integration-infrastructure\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003eContinuous-Integration-Infrastructure\u003c\/a\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_46\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bzimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e bzimport\u003c\/a\u003e set Reference to bz70597.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_47\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bzimport\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e bzimport\u003c\/a\u003e added a subscriber: \u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003eUnknown Object (MLST)\u003c\/span\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/sdb5e3nkynpf4swmwiza\/PHID-FILE-cvbkyayzlwbwsqgg7q4f\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/greg\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"747920\" id=\"747920\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_50\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/greg\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003egreg\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747920\" data-sigil=\"has-tooltip\" data-meta=\"0_49\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 2:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 02:48:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_60\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_61\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"747925\" id=\"747925\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_59\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747925\" data-sigil=\"has-tooltip\" data-meta=\"0_58\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 3:04 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 03:04:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_56\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_57\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_10\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis happens once in a while. It's some sort of deadlock in Jenkins itself. Here's how I generally try to resolve it:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTake deployment-bastion offline in Jenkins \u003ca href=\"https:\/\/integration.wikimedia.org\/ci\/computer\/deployment-bastion.eqiad\/markOffline\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/integration.wikimedia.org\/ci\/computer\/deployment-bastion.eqiad\/markOffline\u003c\/a\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eKill any jenkins jobs running on deployment-bastion via Jenkins UI\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eKill all pending jobs in the Jenkins queue that are "waiting on executors"\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisconnect deployment-bastion \u003ca href=\"https:\/\/integration.wikimedia.org\/ci\/computer\/deployment-bastion.eqiad\/disconnect\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/integration.wikimedia.org\/ci\/computer\/deployment-bastion.eqiad\/disconnect\u003c\/a\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eBring deployment-bastion back online (button labeled "Bring this node back online")\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eLaunch slave agent (there's a button that says this)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCheck agent log to see that it connected \u003ca href=\"https:\/\/integration.wikimedia.org\/ci\/computer\/deployment-bastion.eqiad\/log\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/integration.wikimedia.org\/ci\/computer\/deployment-bastion.eqiad\/log\u003c\/a\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSometimes you have to do this whole dance twice before Jenkins realizes that the there are a bunch of executors that it can use.\u003c\/p\u003e\n\n\u003cp\u003eThis deadlock seems to happen more often than not following or during a database update that is taking a while to complete.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/bxdymb7zqtipuiytbor5\/PHID-FILE-hh7qwoflwjmrunovgr5a\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Jdforrester-WMF\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_71\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_72\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/12\/\" data-sigil=\"has-tooltip\" data-meta=\"0_73\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_74\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"747931\" id=\"747931\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_70\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Jdforrester-WMF\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003eJdforrester-WMF\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747931\" data-sigil=\"has-tooltip\" data-meta=\"0_69\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 4:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 04:58:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_67\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_68\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_12\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks! Should we write these down somewhere for the next time it occurs?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_84\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_85\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"747944\" id=\"747944\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_83\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747944\" data-sigil=\"has-tooltip\" data-meta=\"0_82\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 5:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 05:00:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_80\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_14\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e(In reply to James Forrester from comment #2)\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eThanks! Should we write these down somewhere for the next time it occurs?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eGreg was way ahead of you -- \u003ca href=\"https:\/\/www.mediawiki.org\/w\/index.php?title=Continuous_integration\/Jenkins&diff=1144517&oldid=1091321\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/www.mediawiki.org\/w\/index.php?title=Continuous_integration\/Jenkins&diff=1144517&oldid=1091321\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/bxdymb7zqtipuiytbor5\/PHID-FILE-hh7qwoflwjmrunovgr5a\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Jdforrester-WMF\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_95\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/12\/\" data-sigil=\"has-tooltip\" data-meta=\"0_97\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_98\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"747959\" id=\"747959\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Jdforrester-WMF\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003eJdforrester-WMF\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747959\" data-sigil=\"has-tooltip\" data-meta=\"0_93\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 5:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 05:03:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_91\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_92\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_16\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHa. Thanks, both!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/umt27gx6hlb6rcfsp32a\/PHID-FILE-q3w2rgnjujl5zn4nnmmd\/alphanumeric_aleo-white_C.png-_984060-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Cmcmahon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"747977\" id=\"747977\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Cmcmahon\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Cmcmahon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747977\" data-sigil=\"has-tooltip\" data-meta=\"0_106\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 4:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 16:19:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_104\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_18\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMeta: one of the themes for RelEng coming up is improving Jenkins performance, this list of issues is a good place to start.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_119\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"747986\" id=\"747986\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#747986\" data-sigil=\"has-tooltip\" data-meta=\"0_115\"\u003e\u003cspan class=\"screen-only\"\u003eSep 9 2014, 11:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-09 23:46:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_113\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_20\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis happened again today. It seems like it always involves the database update job. Maybe related to \u003ca href=\"https:\/\/issues.jenkins-ci.org\/browse\/JENKINS-10944\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/issues.jenkins-ci.org\/browse\/JENKINS-10944\u003c\/a\u003e. Apparently the parent build for a matrix job is not supposed to occupy an executor, but sometimes it does. Seems to be an open bug still upstream.\u003c\/p\u003e\n\n\u003cp\u003eI have manually changed the config for the beta-update-databases-eqiad job in Jenkins to use the "Throttle Concurrent Builds" in an attempt to keep Jenkins from confusing itself. If this "works" then the changes should be backported into the JJB configuration to keep it from coming back the next time the job is updated from config.\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMaximum Total Concurrent Builds: 2\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMaximum Concurrent Builds Per Node: 2\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThrottle Matrix master builds: true\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThrottle Matrix configuration builds: true\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e(reopening to keep this on my radar for a few days)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_128\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/sdb5e3nkynpf4swmwiza\/PHID-FILE-cvbkyayzlwbwsqgg7q4f\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/greg\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748002\" id=\"748002\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/greg\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003egreg\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748002\" data-sigil=\"has-tooltip\" data-meta=\"0_126\"\u003e\u003cspan class=\"screen-only\"\u003eSep 23 2014, 10:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-23 22:55:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_124\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_22\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSept 23rd: 22:51 bd808: Jenkins stuck trying to update database in beta again with the dumb "waiting for executors" bug\/problem\u003c\/p\u003e\n\n\u003cp\u003e(just keeping a log of when it occurs)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748010\" id=\"748010\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748010\" data-sigil=\"has-tooltip\" data-meta=\"0_135\"\u003e\u003cspan class=\"screen-only\"\u003eSep 23 2014, 11:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-09-23 23:18:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_133\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_24\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e(In reply to Bryan Davis from comment #6)\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI have manually changed the config for the beta-update-databases-eqiad job\u003cbr \/\u003e\nin Jenkins to use the "Throttle Concurrent Builds" in an attempt to keep\u003cbr \/\u003e\nJenkins from confusing itself. If this "works" then the changes should be\u003cbr \/\u003e\nbackported into the JJB configuration to keep it from coming back the next\u003cbr \/\u003e\ntime the job is updated from config.\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMaximum Total Concurrent Builds: 2\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eMaximum Concurrent Builds Per Node: 2\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThrottle Matrix master builds: true\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThrottle Matrix configuration builds: true\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e(reopening to keep this on my radar for a few days)\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThe throttle settings are still in place, so they didn't fix the problem. The fact that we didn't see this error for a couple of weeks is apparently uncorrelated.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_152\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_150\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_151\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748028\" id=\"748028\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748028\" data-sigil=\"has-tooltip\" data-meta=\"0_146\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2014, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-10-24 14:57:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_144\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_26\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat happened again on Oct 23 2014.  Looking at Jenkins thread dumps, the deployment-bastion execute thread are locked by the Gearman plugin.   It must have some logic error somewhere but I can't really debug Java ;(\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_163\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_161\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748039\" id=\"748039\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_160\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748039\" data-sigil=\"has-tooltip\" data-meta=\"0_159\"\u003e\u003cspan class=\"screen-only\"\u003eOct 31 2014, 12:21 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-10-31 00:21:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_157\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_28\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eUn-licking this cookie. I took a shot at it but the problem seems to be deeper in the stack.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_176\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_174\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748051\" id=\"748051\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748051\" data-sigil=\"has-tooltip\" data-meta=\"0_170\"\u003e\u003cspan class=\"screen-only\"\u003eOct 31 2014, 1:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-10-31 01:03:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_168\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_30\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI forgot to update this bug after my debugging session on Oct 24th here is the rough brain dump:\u003c\/p\u003e\n\n\u003cp\u003eThe Gearman plugin source code is at \u003ca href=\"https:\/\/review.openstack.org\/openstack-infra\/gearman-plugin.git\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/review.openstack.org\/openstack-infra\/gearman-plugin.git\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eI have added a logger for the Gearman plugin ( hudson.plugins.gearman.logger @ INFO ):\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/integration.wikimedia.org\/ci\/log\/Plugins%20-%20Gearman\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/integration.wikimedia.org\/ci\/log\/Plugins%20-%20Gearman\/\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eWhenever the issue occurs we can switch it to FINE and get some debug messages.\u003c\/p\u003e\n\n\u003cp\u003eIIRC the bastion executors threads were held in lock() within src\/main\/java\/hudson\/plugins\/gearman\/NodeAvailabilityMonitor.java lock() function. It takes a worker as parameter and has a wait(5000).\u003c\/p\u003e\n\n\u003cp\u003eI think the debug message had a 'null' worker possibly ending in a dead end of being always considered busy and thus waiting.\u003c\/p\u003e\n\n\u003cp\u003eThat might be a bad interaction with the jobs that are scheduled by Jenkins itself such as the hourly database update that runs on deployment bastion as well.\u003c\/p\u003e\n\n\u003cp\u003eAnd of course: I have no idea how to reproduce it nor how to hook a debugger in Jenkins nor how to dump the state of variables :-\/\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_190\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_186\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_187\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_188\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_189\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748059\" id=\"748059\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_185\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#748059\" data-sigil=\"has-tooltip\" data-meta=\"0_184\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2014, 2:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-04 14:22:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_182\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_183\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_32\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCreated attachment 17012\u003cbr \/\u003e\nRaw thread dump from Jenkins\u003c\/p\u003e\n\n\u003cp\u003edeployment-bastion.eqiad node is in a deadlock right now, I took a thread dump from Jenkins using \u003ca href=\"https:\/\/integration.wikimedia.org\/ci\/monitoring?part=threadsDump\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/integration.wikimedia.org\/ci\/monitoring?part=threadsDump\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eThe Jenkins Gearman plugin seems to lock the node executors, with threads looking like:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e  "Gearman worker deployment-bastion.eqiad_exec-0" daemon prio=5 TIMED_WAITING\n\tjava.lang.Object.wait(Native Method)\n\thudson.plugins.gearman.NodeAvailabilityMonitor.lock(NodeAvailabilityMonitor.java:83)\n\thudson.plugins.gearman.MyGearmanWorkerImpl.sendGrabJob(MyGearmanWorkerImpl.java:388)\n\thudson.plugins.gearman.MyGearmanWorkerImpl.handleSessionEvent(MyGearmanWorkerImpl.java:422)\n\thudson.plugins.gearman.MyGearmanWorkerImpl$GrabJobEventHandler.handleEvent(MyGearmanWorkerImpl.java:99)\n\torg.gearman.common.GearmanTask.handleGearmanIOEvent(GearmanTask.java:92)\n\torg.gearman.common.GearmanJobServerSession.handleResSessionEvent(GearmanJobServerSession.java:329)\n\torg.gearman.common.GearmanJobServerSession.handleSessionEvent(GearmanJobServerSession.java:240)\n\torg.gearman.common.GearmanJobServerSession.driveSessionIO(GearmanJobServerSession.java:203)\n\thudson.plugins.gearman.MyGearmanWorkerImpl.work(MyGearmanWorkerImpl.java:357)\n\thudson.plugins.gearman.AbstractWorkerThread.run(AbstractWorkerThread.java:166)\n\tjava.lang.Thread.run(Thread.java:701)\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003eAttached\u003c\/strong\u003e: \u003cdiv href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/kwm3sonlkm7vi7f2kg3q\/PHID-FILE-zjrilyqodsbp3r4vmn5n\/jenkins-threads-dump.txt\" target=\"_blank\" rel=\"noreferrer\" class=\"phabricator-remarkup-embed-layout-link \" data-sigil=\"lightboxable\" data-meta=\"0_1\" data-mustcapture=\"1\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file-text-o phabricator-remarkup-embed-layout-icon\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003cspan class=\"phabricator-remarkup-embed-layout-info-block\"\u003e\u003cspan class=\"phabricator-remarkup-embed-layout-name\"\u003ejenkins-threads-dump.txt\u003c\/span\u003e\u003cspan class=\"phabricator-remarkup-embed-layout-info\"\u003e135 KB\u003c\/span\u003e\u003c\/span\u003e\u003ca class=\"phabricator-remarkup-embed-layout-download\" href=\"https:\/\/phab.wmfusercontent.org\/file\/download\/kwm3sonlkm7vi7f2kg3q\/PHID-FILE-zjrilyqodsbp3r4vmn5n\/jenkins-threads-dump.txt\"\u003eDownload\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_204\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_200\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_202\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748066\" id=\"748066\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_199\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#748066\" data-sigil=\"has-tooltip\" data-meta=\"0_198\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2014, 2:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-04 14:24:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_196\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_34\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFrom Jenkins log \u003ca href=\"https:\/\/integration.wikimedia.org\/ci\/log\/Plugins%20-%20Gearman\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/integration.wikimedia.org\/ci\/log\/Plugins%20-%20Gearman\/\u003c\/a\u003e  ( hudson.plugins.gearman.logger @ FINEST ) there is a spam of:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eAvailabilityMonitor canTake request for deployment-bastion.eqiad_exec-1\nNov 04, 2014 2:15:19 PM FINE hudson.plugins.gearman.NodeAvailabilityMonitor canTake\nAvailabilityMonitor canTake request for deployment-bastion.eqiad_exec-1\nNov 04, 2014 2:15:19 PM FINE hudson.plugins.gearman.NodeAvailabilityMonitor canTake\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_217\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_213\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_214\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_215\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748078\" id=\"748078\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748078\" data-sigil=\"has-tooltip\" data-meta=\"0_211\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2014, 2:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-04 14:30:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_209\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_36\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe rest of the executors are hold in lock with:\u003c\/p\u003e\n\n\u003cp\u003e"Gearman worker deployment-bastion.eqiad_exec-2"\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003edaemon prio=5 TIMED_WAITING\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ejava.lang.Object.wait(Native Method)\u003cbr \/\u003e\n hudson.plugins.gearman\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e.NodeAvailabilityMonitor.lock(NodeAvailabilityMonitor.java:83)\n.MyGearmanWorkerImpl.sendGrabJob(MyGearmanWorkerImpl.java:388)\n.MyGearmanWorkerImpl.handleSessionEvent(MyGearmanWorkerImpl.java:422)\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_230\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_228\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748091\" id=\"748091\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748091\" data-sigil=\"has-tooltip\" data-meta=\"0_224\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2014, 4:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-04 16:22:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_222\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_38\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI disconnected the Jenkins Gearman client  which is holding the lock. That unleashed the jobs that are meant to be run on deployment-bastion.eqiad.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_243\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_239\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_241\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748104\" id=\"748104\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_238\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748104\" data-sigil=\"has-tooltip\" data-meta=\"0_237\"\u003e\u003cspan class=\"screen-only\"\u003eNov 5 2014, 10:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-05 10:12:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_235\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_40\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI am moving this bug to Continous integration since its root cause is somewhere in Jenkins Gearman plugin. Will make it easier for me to find that bug again.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_256\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_254\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"748120\" id=\"748120\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#748120\" data-sigil=\"has-tooltip\" data-meta=\"0_250\"\u003e\u003cspan class=\"screen-only\"\u003eNov 5 2014, 10:18 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2014-11-05 10:18:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_248\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_42\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMost executors are held in lock as mentioned on comment #14.  I noticed one thread having a different stacktrace:\u003c\/p\u003e\n\n\u003cp\u003e"Gearman worker deployment-bastion.eqiad_exec-1" daemon prio=5 WAITING\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ejava.lang.Object.wait(Native Method)\njava.lang.Object.wait(Object.java:502)\nhudson.remoting.AsyncFutureImpl.get(AsyncFutureImpl.java:73)\nhudson.plugins.gearman\n    .StartJobWorker.safeExecuteFunction(StartJobWorker.java:196)\nhudson.plugins.gearman\n    .StartJobWorker.executeFunction(StartJobWorker.java:114)\norg.gearman.worker\n    .AbstractGearmanFunction.call(AbstractGearmanFunction.java:125)\norg.gearman.worker\n    .AbstractGearmanFunction.call(AbstractGearmanFunction.java:22)\nhudson.plugins.gearman.MyGearmanWorkerImpl\n    .submitFunction(MyGearmanWorkerImpl.java:590)\nhudson.plugins.gearman.MyGearmanWorkerImpl\n    .work(MyGearmanWorkerImpl.java:374)\nhudson.plugins.gearman.AbstractWorkerThread\n    .run(AbstractWorkerThread.java:166)\njava.lang.Thread.run(Thread.java:701)\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-ynivjflmc2dcl6w5ut5v"}},{"phid":"PHID-FILE-zjrilyqodsbp3r4vmn5n","viewable":false,"uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/kwm3sonlkm7vi7f2kg3q\/PHID-FILE-zjrilyqodsbp3r4vmn5n\/jenkins-threads-dump.txt","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/kwm3sonlkm7vi7f2kg3q\/PHID-FILE-zjrilyqodsbp3r4vmn5n\/jenkins-threads-dump.txt","name":"jenkins-threads-dump.txt","monogram":"F14761","icon":"fa-file-text-o","size":"135 KB"},[],{"hovercardSpec":{"objectPHID":"PHID-USER-ynivjflmc2dcl6w5ut5v"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-6wswany56ulk4z7k33jb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ynivjflmc2dcl6w5ut5v"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ynivjflmc2dcl6w5ut5v"}},{"hovercardSpec":{"objectPHID":"PHID-MLST-3n6f7yggtuz4ktrvskho"}},{"hovercardSpec":{"objectPHID":"PHID-USER-z3l4i7dl52qicxtephy5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-oisrjbvd6au5nkt"},{"hovercardSpec":{"objectPHID":"PHID-USER-ydswvwhh5pm4lshahjje"}},{"phid":"PHID-XACT-TASK-jf7o2v2jx2fzwwn"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-syueutmwmmedfwx"},{"hovercardSpec":{"objectPHID":"PHID-USER-ydswvwhh5pm4lshahjje"}},{"phid":"PHID-XACT-TASK-eelws3cxdsvosvb"},{"hovercardSpec":{"objectPHID":"PHID-USER-kqibbfgfpgocyzwe32lv"}},{"phid":"PHID-XACT-TASK-bgejj6njzmlejyp"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-wzflpfnvshmv5ve"},{"hovercardSpec":{"objectPHID":"PHID-USER-z3l4i7dl52qicxtephy5"}},{"phid":"PHID-XACT-TASK-j74c2cwo4ja7pao"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-ehaofycejj46x2w"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-e7x4a2js5rukgk3"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-hkfm4l2cojkknca"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-jpuqddvloo6iemh"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-tidhzslja4ht6on"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-tal3zsbxcvaotfv"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-gz6hro3zs6zjtu2"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-ykxwbpubcepzppp"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-e2pew5kfs7gmo4f"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-7a3pwp7lv4dlxi4"},{"tip":"Via Conduit"},[],[],[],[],{"phid":"PHID-XACT-TASK-hdsgxop4hj2o2li","anchor":"747905"},{"tip":"Via Old World"},[],{"phid":"PHID-XACT-TASK-269290c8b7d9055","anchor":"747920"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-oisrjbvd6au5nkt\/","ref":"T72597#747925"},[],{"anchor":"747925"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_52\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_53\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-oisrjbvd6au5nkt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_54\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_55\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-oisrjbvd6au5nkt","anchor":"747925"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jf7o2v2jx2fzwwn\/","ref":"T72597#747931"},[],{"anchor":"747931"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_63\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_64\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jf7o2v2jx2fzwwn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_65\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_66\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"ResourceLoader module terminator ","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-jf7o2v2jx2fzwwn","anchor":"747931"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-syueutmwmmedfwx\/","ref":"T72597#747944"},[],{"anchor":"747944"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_76\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_77\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-syueutmwmmedfwx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_78\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-syueutmwmmedfwx","anchor":"747944"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-eelws3cxdsvosvb\/","ref":"T72597#747959"},[],{"anchor":"747959"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_87\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eelws3cxdsvosvb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_89\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"ResourceLoader module terminator ","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-eelws3cxdsvosvb","anchor":"747959"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bgejj6njzmlejyp\/","ref":"T72597#747977"},[],{"anchor":"747977"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_100\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bgejj6njzmlejyp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_102\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bgejj6njzmlejyp","anchor":"747977"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wzflpfnvshmv5ve\/","ref":"T72597#747986"},[],{"anchor":"747986"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_109\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_110\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wzflpfnvshmv5ve\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_111\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_112\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-wzflpfnvshmv5ve","anchor":"747986"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-j74c2cwo4ja7pao\/","ref":"T72597#748002"},[],{"anchor":"748002"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_120\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_121\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j74c2cwo4ja7pao\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_122\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-j74c2cwo4ja7pao","anchor":"748002"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ehaofycejj46x2w\/","ref":"T72597#748010"},[],{"anchor":"748010"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_129\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ehaofycejj46x2w\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_131\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ehaofycejj46x2w","anchor":"748010"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-e7x4a2js5rukgk3\/","ref":"T72597#748028"},[],{"anchor":"748028"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_140\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_141\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-e7x4a2js5rukgk3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_142\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-e7x4a2js5rukgk3","anchor":"748028"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hkfm4l2cojkknca\/","ref":"T72597#748039"},[],{"anchor":"748039"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_153\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_154\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hkfm4l2cojkknca\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_155\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-hkfm4l2cojkknca","anchor":"748039"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jpuqddvloo6iemh\/","ref":"T72597#748051"},[],{"anchor":"748051"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_164\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_165\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jpuqddvloo6iemh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_166\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_167\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-jpuqddvloo6iemh","anchor":"748051"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tidhzslja4ht6on\/","ref":"T72597#748059"},[],{"anchor":"748059"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_177\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_178\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tidhzslja4ht6on\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_179\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_180\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-tidhzslja4ht6on\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_181\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tidhzslja4ht6on","anchor":"748059"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tal3zsbxcvaotfv\/","ref":"T72597#748066"},[],{"anchor":"748066"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_191\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_192\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tal3zsbxcvaotfv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_193\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_194\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-tal3zsbxcvaotfv\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tal3zsbxcvaotfv","anchor":"748066"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gz6hro3zs6zjtu2\/","ref":"T72597#748078"},[],{"anchor":"748078"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_205\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_206\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gz6hro3zs6zjtu2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_207\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_208\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-gz6hro3zs6zjtu2","anchor":"748078"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ykxwbpubcepzppp\/","ref":"T72597#748091"},[],{"anchor":"748091"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_218\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_219\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ykxwbpubcepzppp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_220\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_221\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ykxwbpubcepzppp","anchor":"748091"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-e2pew5kfs7gmo4f\/","ref":"T72597#748104"},[],{"anchor":"748104"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_231\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_232\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-e2pew5kfs7gmo4f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_233\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-e2pew5kfs7gmo4f","anchor":"748104"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7a3pwp7lv4dlxi4\/","ref":"T72597#748120"},[],{"anchor":"748120"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_244\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7a3pwp7lv4dlxi4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_246\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-7a3pwp7lv4dlxi4","anchor":"748120"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/602387e0\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}