for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6nxmcvti523imj5h3q4y\/PHID-FILE-pg4gojgb3dch6ndeqdow\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Diannaa\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5889922\" id=\"5889922\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_42\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Diannaa\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eDiannaa\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5889922\" data-sigil=\"has-tooltip\" data-meta=\"0_41\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 12:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 12:37:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5889930\" id=\"5889930\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_45\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_6\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/3534219\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5889930\" data-sigil=\"has-tooltip\" data-meta=\"0_44\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 12:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 12:37:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_56\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/pdvgf6bwbegtkliymkn2\/PHID-FILE-xratrbg4u35mp2lcxckv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Masumrezarock100\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890012\" id=\"5890012\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_54\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Masumrezarock100\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003eMasumrezarock100\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/tools\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003eTools\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890012\" data-sigil=\"has-tooltip\" data-meta=\"0_53\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 1:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 13:17:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_55\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Masumrezarock100\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003eMasumrezarock100\u003c\/a\u003e subscribed.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_51\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_52\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_10\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI will ping Erwig on Discord.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c5mll4i2ivmjlggdpizk\/PHID-FILE-x2xqxc36bpfpfub7m6rd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Aklapper\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/2\/\" data-sigil=\"has-tooltip\" data-meta=\"0_65\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-bug\" data-meta=\"0_66\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890021\" id=\"5890021\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_64\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003eAklapper\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890021\" data-sigil=\"has-tooltip\" data-meta=\"0_63\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 1:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 13:25:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_61\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_62\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_13\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSee \u003ca href=\"\/T243736\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT243736\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-indigo\" data-sigil=\"transaction anchor-container\" data-meta=\"0_70\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c5mll4i2ivmjlggdpizk\/PHID-FILE-x2xqxc36bpfpfub7m6rd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Aklapper\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5890024\" id=\"5890024\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-indigo\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-check phui-timeline-icon\" data-meta=\"0_69\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003eAklapper\u003c\/a\u003e closed this task as a duplicate of \u003ca href=\"\/T243736\" class=\"phui-handle handle-status-closed\"\u003eT243736:  Earwig's Copyvio Detector down due to "The supplied API key is not configured for use from this IP address." from google-api-proxy\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890024\" data-sigil=\"has-tooltip\" data-meta=\"0_68\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 1:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 13:25:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/pdvgf6bwbegtkliymkn2\/PHID-FILE-xratrbg4u35mp2lcxckv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Masumrezarock100\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890028\" id=\"5890028\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_78\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Masumrezarock100\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003eMasumrezarock100\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890028\" data-sigil=\"has-tooltip\" data-meta=\"0_77\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 1:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 13:26:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_75\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_76\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_16\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMeh, it seems he is not available currently on Discord. It would be faster if he was.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/pdvgf6bwbegtkliymkn2\/PHID-FILE-xratrbg4u35mp2lcxckv\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Masumrezarock100\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890050\" id=\"5890050\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-exclamation-circle phui-timeline-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Masumrezarock100\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003eMasumrezarock100\u003c\/a\u003e reopened this task as \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#5890050\" data-sigil=\"has-tooltip\" data-meta=\"0_87\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 1:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 13:31:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_85\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_86\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_18\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNo, this is not the same problem. I am seeing a 504 gateway timeout error if I visit \u003ca href=\"https:\/\/tools.wmflabs.org\/copyvios\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/tools.wmflabs.org\/copyvios\/\u003c\/a\u003e while \u003ca href=\"\/T243736\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT243736\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e describes the error message as\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eAn error occurred while using the search engine (Google Error: HTTP Error 403: Forbidden). Note: there is a daily limit on the number of search queries the tool is allowed to make. You may repeat the check without using the search engine.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI remember I used it yesterday and it was working but it is down at the moment.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kbmes7vod7dr6zxwq4y6\/PHID-FILE-utqg6b6ebxt7phmod4z7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Curb_Safe_Charmer\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890119\" id=\"5890119\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_98\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Curb_Safe_Charmer\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003eCurb_Safe_Charmer\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#5890119\" data-sigil=\"has-tooltip\" data-meta=\"0_97\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 1:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 13:40:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_95\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_19\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI raised this on IRC in #wikimedia-cloud earlier. \u003cspan class=\"phabricator-remarkup-mention-unknown\"\u003e@arturo\u003c\/span\u003e checked the logs and found \u003ctt class=\"remarkup-monospaced\"\u003euWSGI listen queue of socket ":8000" (fd: 7) full !!! (101\/100)\u003c\/tt\u003e and restarted the web service.\u003c\/p\u003e\n\n\u003cp\u003eHe also found the following error:\u003c\/p\u003e\n\n\u003cp\u003eopen("\/usr\/lib\/uwsgi\/plugins\/python3_plugin.so"): No such file or directory [core\/utils.c line 3664]\u003cbr \/\u003e\n\u003cspan class=\"remarkup-highlight\"\u003e! UNABLE to load uWSGI plugin: \/usr\/lib\/uwsgi\/plugins\/python3_plugin.so: cannot open shared object file: No such file or directory !\u003c\/span\u003e\u003cbr \/\u003e\n[uWSGI] getting INI configuration from \/data\/project\/copyvios\/www\/python\/uwsgi.ini\u003c\/p\u003e\n\n\u003cp\u003eHe suggested that the metrics at \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/toolforge-k8s-namespace-resources\/kubernetes-namespace-resources?orgId=1&var-namespace=tool-copyvios&refresh=5m&from=now-1h&to=now\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana-labs.wikimedia.org\/d\/toolforge-k8s-namespace-resources\/kubernetes-namespace-resources?orgId=1&var-namespace=tool-copyvios&refresh=5m&from=now-1h&to=now\u003c\/a\u003e indicated it needed higher CPU limits.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wvfbttfzqavdnhl2ey5u\/PHID-FILE-jx5rgnmgk43jovj7rcji\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Earwig\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890559\" id=\"5890559\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Earwig\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003eEarwig\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003ebd808\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890559\" data-sigil=\"has-tooltip\" data-meta=\"0_106\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 4:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 16:34:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_104\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_23\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI'm not sure how to raise the limits beyond where they currently are (1 vCPU and 4GiB of RAM). Because we didn't have this issue with the old k8s cluster, I'm also not sure what has changed now. \u003ca href=\"\/p\/bd808\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@bd808\u003c\/span\u003e\u003c\/a\u003e, any ideas?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wvfbttfzqavdnhl2ey5u\/PHID-FILE-jx5rgnmgk43jovj7rcji\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Earwig\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890561\" id=\"5890561\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Earwig\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eEarwig\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890561\" data-sigil=\"has-tooltip\" data-meta=\"0_115\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 4:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 16:36:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_113\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAlso, the error Arturo found is not a concern. That will happen for any Python 2 webservice because, as I understand it, the container image only makes Python 2 available but sets uwsgi up to load both the Python 2 and 3 plugins.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_126\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wvfbttfzqavdnhl2ey5u\/PHID-FILE-jx5rgnmgk43jovj7rcji\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Earwig\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5890980\" id=\"5890980\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Earwig\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eEarwig\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5890980\" data-sigil=\"has-tooltip\" data-meta=\"0_124\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 7:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 19:11:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_122\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe CPU usage being reported by grafana does not seem correct. Inside the container, with \u003ctt class=\"remarkup-monospaced\"\u003eps\u003c\/tt\u003e, I see:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003etools.copyvios@:~$ ps auxwwf\nUSER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND\ntools.c+   102  0.4  0.0  44308  4192 pts\/1    Ss   19:08   0:00 \/bin\/bash\ntools.c+   115  0.0  0.0  41304  2700 pts\/1    R+   19:09   0:00  \\_ ps auxwwf\ntools.c+    39  0.0  0.0  44312  4104 pts\/0    Ss+  16:41   0:00 \/bin\/bash\ntools.c+     1  0.0  0.8 208688 68936 ?        Ss   16:38   0:08 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\ntools.c+     9  3.8  2.5 969832 210368 ?       Sl   16:39   5:44 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\ntools.c+    10  3.7  2.7 963352 220660 ?       Sl   16:39   5:34 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\ntools.c+    11  3.1  2.4 976900 196780 ?       Sl   16:39   4:39 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\ntools.c+    12  7.4  4.1 1026328 335184 ?      Sl   16:39  11:09 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\ntools.c+    14  9.5  3.1 985556 259056 ?       Sl   16:39  14:17 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\ntools.c+    92  4.0  1.4 943992 121220 ?       Sl   19:02   0:16 \/usr\/bin\/uwsgi --plugin python,python3 --http-socket :8000 --chdir \/data\/project\/copyvios\/www\/python\/src --logto \/data\/project\/copyvios\/uwsgi.log --callable app --manage-script-name --workers 4 --mount \/copyvios=\/data\/project\/copyvios\/www\/python\/src\/app.py --die-on-term --strict --master --venv \/data\/project\/copyvios\/www\/python\/venv --ini \/data\/project\/copyvios\/www\/python\/uwsgi.ini\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThat's about 30% usage, which \u003ctt class=\"remarkup-monospaced\"\u003etop\u003c\/tt\u003e agrees with. However, grafana reports usage consistently around 100% for the past hour: \u003ca href=\"https:\/\/grafana-labs.wikimedia.org\/d\/toolforge-k8s-namespace-resources\/kubernetes-namespace-resources?orgId=1&var-namespace=tool-copyvios&refresh=5s&from=1581963062073&to=1581966662073&panelId=1&fullscreen\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana-labs.wikimedia.org\/d\/toolforge-k8s-namespace-resources\/kubernetes-namespace-resources?orgId=1&var-namespace=tool-copyvios&refresh=5s&from=1581963062073&to=1581966662073&panelId=1&fullscreen\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/DavidBrooks\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5891195\" id=\"5891195\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/DavidBrooks\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eDavidBrooks\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5891195\" data-sigil=\"has-tooltip\" data-meta=\"0_133\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 8:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 20:50:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_131\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_28\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eLooking (for the first time) at that grafana graph, and due respect to your 30% calculation: the usage reported there was spiky for the first ~9 days, but went to an almost-solid 100% about them time we noticed the timeouts.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_144\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/DavidBrooks\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5891465\" id=\"5891465\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/DavidBrooks\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eDavidBrooks\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5891465\" data-sigil=\"has-tooltip\" data-meta=\"0_142\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2020, 11:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-17 23:37:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_140\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_141\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt came back briefly, but it's gone again. From where I am, it connects to the server and then 504's after 3 minutes.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wvfbttfzqavdnhl2ey5u\/PHID-FILE-jx5rgnmgk43jovj7rcji\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Earwig\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5891692\" id=\"5891692\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Earwig\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eEarwig\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5891692\" data-sigil=\"has-tooltip\" data-meta=\"0_151\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 18 2020, 2:57 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-18 02:57:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_149\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_150\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have hacked my way to a higher apparent memory\/CPU limit by adjusting the container resource requests to match the command-line limits instead of half (so 4 GiB RAM and 1 CPU instead of 2 GiB and 0.5, respectively). While this is certainly not a good long-term solution, I want to see if the tool behaves better under these conditions.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/wvfbttfzqavdnhl2ey5u\/PHID-FILE-jx5rgnmgk43jovj7rcji\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Earwig\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5891797\" id=\"5891797\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Earwig\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eEarwig\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5891797\" data-sigil=\"has-tooltip\" data-meta=\"0_160\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 18 2020, 4:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-18 04:33:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_158\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_35\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eInitial results don't seem especially promising; the spiky CPU usage continues for about half an hour after restarting until reaching 100% and staying there seemingly indefinitely. It's not clear to me why this would happen. One conclusion I can make is that it's not directly related to memory usage; I had a theory earlier that we were bumping into the memory quota and swap thrashing, but I don't think this is happening. Memory usage is only around 600MB out of 4GB (per grafana) when we get stuck.\u003c\/p\u003e\n\n\u003cp\u003eI tried to debug one worker that seemed to be holding on to the CPU, but it's difficult. \u003ctt class=\"remarkup-monospaced\"\u003egdb\u003c\/tt\u003e is not available inside the pod, so to get a stack trace I made it dump core, but I can't really analyze the core dump because \u003ctt class=\"remarkup-monospaced\"\u003egdb\u003c\/tt\u003e on the bastion doesn't have access to the right binaries\/libraries. uwsgi logs a stack trace, but it's for a random thread that was trying to acquire a lock, so that's not helpful.\u003c\/p\u003e\n\n\u003cp\u003eIt should be possible to solve\/mitigate this by rearchitecting the tool to handle requests asynchronously, but this is a lot of work to get correct and I don't have time to do that right now. We have survived with the current blocking threaded worker model for years; why is it only an issue on the new cluster even with the limits raised so much? Were the limits on the old cluster really that much higher? I'm so perplexed by all of this.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_165\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/de4ombkvtcykafarakxk\/PHID-FILE-trzifh275un3px5rdwmi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/AntiCompositeNumber\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5891826\" id=\"5891826\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/AntiCompositeNumber\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eAntiCompositeNumber\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5891826\" data-sigil=\"has-tooltip\" data-meta=\"0_163\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 18 2020, 5:15 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-18 05:15:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_174\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/DavidBrooks\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5891928\" id=\"5891928\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/DavidBrooks\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eDavidBrooks\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5891928\" data-sigil=\"has-tooltip\" data-meta=\"0_172\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 18 2020, 7:07 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-18 07:07:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_170\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_38\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks for the efforts so far. Hope you can track it down.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/res\/phabricator\/e132bb6a\/rsrc\/image\/avatar.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Chrisw80\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5893505\" id=\"5893505\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_176\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Chrisw80\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eChrisw80\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5893505\" data-sigil=\"has-tooltip\" data-meta=\"0_175\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 18 2020, 4:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-18 16:23:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/viaegzya5e7w4pjpozkp\/PHID-FILE-5ju4ni7jwuwp5tabfz2g\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Coffee\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5894454\" id=\"5894454\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-trophy phui-timeline-icon\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view sprite-tokens tokens-emoji-5 phui-timeline-token\" data-meta=\"0_180\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Coffee\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eCoffee\u003c\/a\u003e awarded a token.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5894454\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 18 2020, 8:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-02-18 20:21:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-zrvqzxlitcj2reeemxtv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-bh6hfrfqcenlgsxzgopn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vadjrsaatxnwawtj4gez"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vadjrsaatxnwawtj4gez"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas","contextPHID":"PHID-TASK-bh6hfrfqcenlgsxzgopn"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-g7ljr7obhy5hgp4qpuiw"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-zywtwi3xlva5ohkdndwb"}},{"phid":"PHID-XACT-TASK-bs7vaudqhlps5b2"},{"hovercardSpec":{"objectPHID":"PHID-USER-g7ljr7obhy5hgp4qpuiw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"phid":"PHID-XACT-TASK-t5pr2zmexadg54n"},{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-g7ljr7obhy5hgp4qpuiw"}},{"phid":"PHID-XACT-TASK-xmju3ukc5qggb2j"},{"hovercardSpec":{"objectPHID":"PHID-USER-g7ljr7obhy5hgp4qpuiw"}},{"phid":"PHID-XACT-TASK-vmj55rlshguui3j"},{"phid":"PHID-XACT-TASK-3tnivqlekokrlbb"},{"hovercardSpec":{"objectPHID":"PHID-USER-kayzz5nz74je4rodlbt6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wzrwloaakl4fx6ytnj2x"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-4s552ef5354r6cr"},{"hovercardSpec":{"objectPHID":"PHID-USER-wzrwloaakl4fx6ytnj2x"}},{"phid":"PHID-XACT-TASK-rgvuqlbhwtt5oae"},{"hovercardSpec":{"objectPHID":"PHID-USER-wzrwloaakl4fx6ytnj2x"}},{"phid":"PHID-XACT-TASK-7optpxi3hvbhrab"},{"phid":"PHID-XACT-TASK-tt5to2tbyuswef6"},{"hovercardSpec":{"objectPHID":"PHID-USER-gkvymsp5qihiqg4cbedb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gkvymsp5qihiqg4cbedb"}},{"phid":"PHID-XACT-TASK-2tnmtgeug73lrba"},{"hovercardSpec":{"objectPHID":"PHID-USER-wzrwloaakl4fx6ytnj2x"}},{"phid":"PHID-XACT-TASK-maf2w3tc7khtw6a"},{"hovercardSpec":{"objectPHID":"PHID-USER-wzrwloaakl4fx6ytnj2x"}},{"phid":"PHID-XACT-TASK-bigwdsz75xmgmqo"},{"hovercardSpec":{"objectPHID":"PHID-USER-kbzm2um2jcqzty52qf5a"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gkvymsp5qihiqg4cbedb"}},{"phid":"PHID-XACT-TASK-pjznxpzci26qj2c"},{"hovercardSpec":{"objectPHID":"PHID-USER-yi7gudj6r5izxi7njox7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7k5txe72hrdwah5q25gw"}},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kjfqm4ofkoxndl3","anchor":"5889922"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-3r62gee2jmodnj4","anchor":"5889930"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bs7vaudqhlps5b2\/","ref":"T245426#5890012"},[],{"anchor":"5890012"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_47\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_48\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bs7vaudqhlps5b2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_49\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_50\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-6in267ocvdqz3h2","anchor":"5890012"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-t5pr2zmexadg54n\/","ref":"T245426#5890021"},[],{"anchor":"5890021"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_57\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_58\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t5pr2zmexadg54n\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_59\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_60\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Bugwrangler","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-t5pr2zmexadg54n","anchor":"5890021"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hojg2rrjncxvva4","anchor":"5890024"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xmju3ukc5qggb2j\/","ref":"T245426#5890028"},[],{"anchor":"5890028"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_71\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_72\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xmju3ukc5qggb2j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_73\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_74\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xmju3ukc5qggb2j","anchor":"5890028"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vmj55rlshguui3j\/","ref":"T245426#5890050"},[],{"anchor":"5890050"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_80\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vmj55rlshguui3j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_82\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_83\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-vmj55rlshguui3j\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4l3ne2tofyjjruc","anchor":"5890050"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3tnivqlekokrlbb\/","ref":"T245426#5890119"},[],{"anchor":"5890119"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_90\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_91\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3tnivqlekokrlbb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_92\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_93\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3tnivqlekokrlbb\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3tnivqlekokrlbb","anchor":"5890119"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4s552ef5354r6cr\/","ref":"T245426#5890559"},[],{"anchor":"5890559"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_100\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4s552ef5354r6cr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_102\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4v2zavqpsmncxyv","anchor":"5890559"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rgvuqlbhwtt5oae\/","ref":"T245426#5890561"},[],{"anchor":"5890561"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_109\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_110\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rgvuqlbhwtt5oae\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_111\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_112\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rgvuqlbhwtt5oae","anchor":"5890561"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7optpxi3hvbhrab\/","ref":"T245426#5890980"},[],{"anchor":"5890980"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_118\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_119\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7optpxi3hvbhrab\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_120\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_121\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7optpxi3hvbhrab","anchor":"5890980"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tt5to2tbyuswef6\/","ref":"T245426#5891195"},[],{"anchor":"5891195"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_127\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_128\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tt5to2tbyuswef6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_129\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tt5to2tbyuswef6","anchor":"5891195"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2tnmtgeug73lrba\/","ref":"T245426#5891465"},[],{"anchor":"5891465"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_136\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_137\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2tnmtgeug73lrba\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_138\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_139\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2tnmtgeug73lrba","anchor":"5891465"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-maf2w3tc7khtw6a\/","ref":"T245426#5891692"},[],{"anchor":"5891692"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_145\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_146\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-maf2w3tc7khtw6a\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_147\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_148\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-maf2w3tc7khtw6a","anchor":"5891692"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bigwdsz75xmgmqo\/","ref":"T245426#5891797"},[],{"anchor":"5891797"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_154\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bigwdsz75xmgmqo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_156\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_157\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bigwdsz75xmgmqo","anchor":"5891797"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bs2rhcy2vrb6rln","anchor":"5891826"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pjznxpzci26qj2c\/","ref":"T245426#5891928"},[],{"anchor":"5891928"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_166\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_167\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pjznxpzci26qj2c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_168\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pjznxpzci26qj2c","anchor":"5891928"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qn7zvgxtqtdfuwi","anchor":"5893505"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-tklfgwl3cvc646i","anchor":"5894454"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}