for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2143705\" id=\"2143705\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_85\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003ejcrespo\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143705\" data-sigil=\"has-tooltip\" data-meta=\"0_84\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 8:32 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 08:32:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2143716\" id=\"2143716\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_10\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/1237816\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143716\" data-sigil=\"has-tooltip\" data-meta=\"0_87\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 8:32 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 08:32:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_98\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2143718\" id=\"2143718\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_97\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003egerritbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143718\" data-sigil=\"has-tooltip\" data-meta=\"0_96\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 8:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 08:33:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_94\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_95\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_12\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 279090 had a related patch set uploaded (by Jcrespo):\u003cbr \/\u003e\nDepool es2019\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/279090\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/279090\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2143719\" id=\"2143719\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_106\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143719\" data-sigil=\"has-tooltip\" data-meta=\"0_105\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 8:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 08:33:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_103\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_104\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_16\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 279090 merged by Jcrespo:\u003cbr \/\u003e\nDepool es2019\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/279090\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/279090\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_118\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_116\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_117\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2143737\" id=\"2143737\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_115\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143737\" data-sigil=\"has-tooltip\" data-meta=\"0_114\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 8:41 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 08:41:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_112\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_113\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_18\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFrom Icinga logs:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e[Wed Mar 23 01:15:52 2016] SERVICE ALERT: es2019;MariaDB Slave SQL: es3;CRITICAL;SOFT;1;Timeout while attempting connection\n[Wed Mar 23 01:15:52 2016] SERVICE ALERT: es2019;SSH;CRITICAL;SOFT;1;Connection timed out\n[Wed Mar 23 01:15:52 2016] SERVICE ALERT: es2019;Check size of conntrack table;CRITICAL;SOFT;1;Timeout while attempting connection\n[Wed Mar 23 01:16:01 2016] SERVICE ALERT: es2019;MariaDB disk space;CRITICAL;SOFT;1;Timeout while attempting connection\n[Wed Mar 23 01:16:02 2016] SERVICE ALERT: es2019;NTP;CRITICAL;SOFT;1;NTP CRITICAL: No response from NTP server\n[Wed Mar 23 01:16:02 2016] SERVICE ALERT: es2019;MariaDB Slave IO: es3;CRITICAL;SOFT;1;Timeout while attempting connection\n[Wed Mar 23 01:16:12 2016] SERVICE ALERT: es2019;salt-minion processes;CRITICAL;SOFT;1;Timeout while attempting connection\n[Wed Mar 23 01:16:12 2016] HOST ALERT: es2019;DOWN;SOFT;1;PING CRITICAL - Packet loss = 100%\n[Wed Mar 23 01:16:13 2016] SERVICE ALERT: es2019;RAID;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:16:22 2016] SERVICE ALERT: es2019;MariaDB Slave Lag: es3;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:16:41 2016] SERVICE ALERT: es2019;puppet last run;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:02 2016] SERVICE ALERT: es2019;configured eth;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:11 2016] SERVICE ALERT: es2019;Disk space;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:11 2016] HOST ALERT: es2019;DOWN;HARD;2;PING CRITICAL - Packet loss = 100%\n[Wed Mar 23 01:17:11 2016] HOST NOTIFICATION: irc;es2019;DOWN;notify-host-by-irc;PING CRITICAL - Packet loss = 100%\n[Wed Mar 23 01:17:11 2016] SERVICE ALERT: es2019;DPKG;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:11 2016] SERVICE ALERT: es2019;dhclient process;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:31 2016] SERVICE ALERT: es2019;mysqld processes;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:41 2016] SERVICE ALERT: es2019;MariaDB Slave SQL: es3;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:41 2016] SERVICE ALERT: es2019;SSH;CRITICAL;HARD;1;Connection timed out\n[Wed Mar 23 01:17:42 2016] SERVICE ALERT: es2019;Check size of conntrack table;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:42 2016] SERVICE ALERT: es2019;MariaDB disk space;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:17:51 2016] SERVICE ALERT: es2019;NTP;CRITICAL;HARD;1;NTP CRITICAL: No response from NTP server\n[Wed Mar 23 01:17:51 2016] SERVICE ALERT: es2019;MariaDB Slave IO: es3;CRITICAL;HARD;1;Timeout while attempting connection\n[Wed Mar 23 01:18:01 2016] SERVICE ALERT: es2019;salt-minion processes;CRITICAL;HARD;1;Timeout while attempting connection\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_130\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_128\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2143753\" id=\"2143753\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2143753\" data-sigil=\"has-tooltip\" data-meta=\"0_126\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 9:04 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 09:04:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_124\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_20\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eRegarding mysql, it recovered ok, but there are certain transactions that mysql has not properly recoded as executed on the binary log but that were committed properly on innodb.\u003c\/p\u003e\n\n\u003cp\u003eSo we:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDepool the server\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDisable replication alerts\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eStart slave, skip duplicate errors - and record those (I predict around 7 duplicate keys)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRecheck thoroughly the whole table before repooling and potential hardware\/power faults\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2143754\" id=\"2143754\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003ejcrespo\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143754\" data-sigil=\"has-tooltip\" data-meta=\"0_131\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 9:04 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 09:04:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_136\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2143803\" id=\"2143803\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_135\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003ejcrespo\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eTriage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003eBacklog\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eDBA\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143803\" data-sigil=\"has-tooltip\" data-meta=\"0_134\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 9:51 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 09:51:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_147\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_145\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_146\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2143818\" id=\"2143818\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_144\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003ejcrespo\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2143818\" data-sigil=\"has-tooltip\" data-meta=\"0_143\"\u003e\u003cspan class=\"screen-only\"\u003eMar 23 2016, 9:53 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-03-23 09:53:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_141\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_28\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've done the first steps, the following errors were skipped after restarting replication on es2018-bin.000044:287914983 (20160323  1:15:31):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eCould not execute Write_rows_v1 event on table wikidatawiki.blobs_cluster25; Duplicate entry '159397294' for key 'PRIM\nCould not execute Write_rows_v1 event on table arwiki.blobs_cluster25; Duplicate entry '5061392' for key 'PRIMARY\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ethe last one is pending:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eRecheck thoroughly the whole table before repooling and potential hardware\/power faults\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_151\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2209968\" id=\"2209968\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eVolans\u003c\/a\u003e claimed this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2209968\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"screen-only\"\u003eApr 15 2016, 2:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-15 14:17:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_150\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eVolans\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eIn progress\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003eDBA\u003c\/a\u003e board.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_154\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2209986\" id=\"2209986\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_153\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003ejcrespo\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2209986\" data-sigil=\"has-tooltip\" data-meta=\"0_152\"\u003e\u003cspan class=\"screen-only\"\u003eApr 15 2016, 2:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-15 14:19:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_165\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_163\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2212261\" id=\"2212261\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2212261\" data-sigil=\"has-tooltip\" data-meta=\"0_161\"\u003e\u003cspan class=\"screen-only\"\u003eApr 16 2016, 5:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-16 17:38:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_159\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_160\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_36\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've run \u003cbr \/\u003e\n\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e: I've run a script to check the tables (took me only few minutes to actually complete it)\u003c\/p\u003e\n\n\u003cp\u003ePlease review it's logic (I'm new to mediawiki table structures) and \u003cstrong\u003edouble check the timeframes\u003c\/strong\u003e, I've picked 1 hour before and after the timestamp you put above.\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI got the list of wikis from \u003ctt class=\"remarkup-monospaced\"\u003eSHOW DATABASES\u003c\/tt\u003e on \u003ctt class=\"remarkup-monospaced\"\u003ees2019\u003c\/tt\u003e and checked that are the same of \u003ctt class=\"remarkup-monospaced\"\u003ees1019\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI've removed \u003ctt class=\"remarkup-monospaced\"\u003emysql\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003einformation_schema\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003etest\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003eperformance_schema\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003eheartbeat\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI'm getting the \u003ctt class=\"remarkup-monospaced\"\u003erev_text_id\u003c\/tt\u003e from \u003ctt class=\"remarkup-monospaced\"\u003erevision\u003c\/tt\u003e between \u003ctt class=\"remarkup-monospaced\"\u003e20160323001500\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003e20160323021500\u003c\/tt\u003e on dbstore1002\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThen I'm getting the \u003ctt class=\"remarkup-monospaced\"\u003eblob_id\u003c\/tt\u003e with \u003ctt class=\"remarkup-monospaced\"\u003eSUBSTRING_INDEX(old_text, '\/', -1)\u003c\/tt\u003e from \u003ctt class=\"remarkup-monospaced\"\u003etext\u003c\/tt\u003e on dbstore1002\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAnd finally I'm comparing \u003ctt class=\"remarkup-monospaced\"\u003eblob_id\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003eMD5(blob_text)\u003c\/tt\u003e between \u003ctt class=\"remarkup-monospaced\"\u003ees1019\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003ees2019\u003c\/tt\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThe script is on \u003ctt class=\"remarkup-monospaced\"\u003eneodymium\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003e\/home\/volans\/es2019\/check.sh\u003c\/tt\u003e, the results are in \u003ctt class=\"remarkup-monospaced\"\u003echeck.log\u003c\/tt\u003e.\u003c\/p\u003e\n\n\u003ch4 class=\"remarkup-header\"\u003ePossible values in \u003ctt class=\"remarkup-monospaced\"\u003echeck.log\u003c\/tt\u003e for each wiki:\u003c\/h4\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003eOK\u003c\/tt\u003e: all blobs id and their MD5 matches\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003eNo revisions to analyze\u003c\/tt\u003e: there were no revisions in that timeframe in the \u003ctt class=\"remarkup-monospaced\"\u003erevision\u003c\/tt\u003e table\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003eUnable to find blob_id for revision ${MIN_PAGE} or ${MAX_PAGE}\u003c\/tt\u003e: should not happen, needs to be checked manually\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003eERROR 1049 (42000): Unknown database '${DB}'\u003c\/tt\u003e: should happen only for DBs missing in dbstore1002\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ctt class=\"remarkup-monospaced\"\u003eFiles ${DB}_es1019.out and ${DB}_es2019.out differ\u003c\/tt\u003e: data corruption happened, to be analyzed manually\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003ch4 class=\"remarkup-header\"\u003eResults:\u003c\/h4\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003e739\u003c\/strong\u003e wikis OK\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003e177\u003c\/strong\u003e wikis without revisions in that timeframe\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003e22\u003c\/strong\u003e wikis missing from dbstore1002 to get the revision id, some are the same of \u003ca href=\"\/T132838\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT132838\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e other not, see complete list below\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003e0\u003c\/strong\u003e wikis unable to find blob_id from text table\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003e0\u003c\/strong\u003e wikis have differences\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003ch4 class=\"remarkup-header\"\u003eList of wikis not found on dbstore1002 to get the revision\u003c\/h4\u003e\n\n\u003cp\u003eLet me know if you want me to find if there were revisions for those in the core DBs.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e10wikipedia\nbawiktionary\nchwikimedia\nde_labswikimedia\nen_flaggedrevs_labswikimedia\nen_labswikimedia\nflaggedrevs_labswikimedia\nliquidthreads_labswikimedia\nnomcomwiki\nptwikimedia\nreaderfeedback_labswikimedia\nstrategyappswiki\nstrategyappwiki\ntenwikipedia\ntlhwiki\ntlhwiktionary\ntokiponawiki\ntokiponawikibooks\ntokiponawikiquote\ntokiponawiktionary\nviwikimedia\nwikidata\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_176\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_174\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2212826\" id=\"2212826\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2212826\" data-sigil=\"has-tooltip\" data-meta=\"0_172\"\u003e\u003cspan class=\"screen-only\"\u003eApr 17 2016, 1:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-17 13:33:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_170\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_38\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis looks great! Upload it "as is" to dbtools or mediawiki maintenance. I would like to parametrize the servers involved to be able to use the masters for s* or other es* servers and other timeframes.\u003c\/p\u003e\n\n\u003cp\u003eHow much does it take to run? if not much (or it can be paralelized), I would extend a bit the time frame (e.g. issues could be before that or when replication was restarted), then we can repool es2019.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_187\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_185\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2213435\" id=\"2213435\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2213435\" data-sigil=\"has-tooltip\" data-meta=\"0_183\"\u003e\u003cspan class=\"screen-only\"\u003eApr 18 2016, 8:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-18 08:23:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_181\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_40\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e I've quickly parametrized the variables and added the time for each wiki check.\u003cbr \/\u003e\nChecking that it was working I found a minor bug in the previous version and two strange things.\u003cbr \/\u003e\nThe bug was considering OK some wikis for which there were no revisions in that timeframe, I've fixed it.\u003c\/p\u003e\n\n\u003cp\u003eThe first strange thing, probably due to my little knowledge of how MediaWiki handles revisions and text tables, is that for some wikis the \u003ctt class=\"remarkup-monospaced\"\u003eold_text\u003c\/tt\u003e IDs are inverted in respect to the \u003ctt class=\"remarkup-monospaced\"\u003erevision\u003c\/tt\u003e ones, see for example:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e-- from dbstore1002\nMariaDB DBSTORE localhost dawiki > SELECT rev_text_id FROM revision WHERE rev_timestamp > '20160322120000' LIMIT 1;\n+-------------+\n| rev_text_id |\n+-------------+\n|     8442989 |\n+-------------+\n1 row in set (0.00 sec)\n\nMariaDB DBSTORE localhost dawiki > SELECT rev_text_id FROM revision WHERE rev_timestamp > '20160324000000' LIMIT 1;\n+-------------+\n| rev_text_id |\n+-------------+\n|     8444931 |  -- GREATER THAN PREVIOUS ONE AS EXPECTED\n+-------------+\n1 row in set (0.00 sec)\n\nMariaDB DBSTORE localhost dawiki > select SUBSTRING_INDEX(old_text, '\/', -1) from text where old_id = 8442989;\n+------------------------------------+\n| SUBSTRING_INDEX(old_text, '\/', -1) |\n+------------------------------------+\n| 1068541                            |\n+------------------------------------+\n1 row in set (0.00 sec)\n\nMariaDB DBSTORE localhost dawiki > select SUBSTRING_INDEX(old_text, '\/', -1) from text where old_id = 8444931;\n+------------------------------------+\n| SUBSTRING_INDEX(old_text, '\/', -1) |\n+------------------------------------+\n| 1068481                            |  -- SMALLER THAN PREVIOUS ONE\n+------------------------------------+\n1 row in set (0.00 sec)\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eFor those cases for now I'm inverting the blob_ids so that it' actually picks up the blobs, but I'm wondering if I'm mistaking something in the logic and checking a different block of blobs.\u003c\/p\u003e\n\n\u003cp\u003eThe second strange thing is that for some wikis there are no blobs in es1019 corresponding to the \u003ctt class=\"remarkup-monospaced\"\u003eold_text\u003c\/tt\u003e IDs, see for example:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e-- from dbstore1002\nMariaDB DBSTORE localhost abwiki > SELECT rev_text_id FROM revision WHERE rev_timestamp > '20160322120000' LIMIT 1;\n+-------------+\n| rev_text_id |\n+-------------+\n|       47125 |\n+-------------+\n1 row in set (0.00 sec)\n\nMariaDB DBSTORE localhost abwiki > SELECT rev_text_id FROM revision WHERE rev_timestamp > '20160324000000' LIMIT 1;\n+-------------+\n| rev_text_id |\n+-------------+\n|       47147 |\n+-------------+\n1 row in set (0.00 sec)\n\nMariaDB DBSTORE localhost abwiki > select SUBSTRING_INDEX(old_text, '\/', -1) from text where old_id = 47125;\n+------------------------------------+\n| SUBSTRING_INDEX(old_text, '\/', -1) |\n+------------------------------------+\n| 4645                               |\n+------------------------------------+\n1 row in set (0.00 sec)\n\nMariaDB DBSTORE localhost abwiki > select SUBSTRING_INDEX(old_text, '\/', -1) from text where old_id = 47147;\n+------------------------------------+\n| SUBSTRING_INDEX(old_text, '\/', -1) |\n+------------------------------------+\n| 4662                               |\n+------------------------------------+\n1 row in set (0.00 sec)\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ebut then on \u003ctt class=\"remarkup-monospaced\"\u003ees1019\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# empty set\n$ mysql -h es1019.eqiad.wmnet -e "SELECT blob_id  FROM blobs_cluster25 WHERE blob_id between 4644 and 4662" abwiki\n$\n\n# trying with wider IDs\n$ mysql -h es1019.eqiad.wmnet -e "SELECT blob_id  FROM blobs_cluster25 WHERE blob_id between 4544 and 7862" abwiki\n+---------+\n| blob_id |\n+---------+\n|    4544 |\n|    4545 |\n|    4546 |\n+---------+\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eWhere are those revisions?\u003c\/p\u003e\n\n\u003cp\u003eGiven those things that I cannot explain can you confirm that the logic is correct?\u003cbr \/\u003e\nI know that the first 4 queries in the script can be combined in one or two, but because I'm not fully sure of the underlying logic of MediaWiki I've keep it like this for now.\u003c\/p\u003e\n\n\u003cp\u003eI'm sending the CR over with what I have so far.\u003c\/p\u003e\n\n\u003cp\u003eThis is the current version results with a slightly larger timeframe:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ time .\/external_storage_consistency_check.sh -s 20160322000000 -e 20160324000000 -r dbstore1002.eqiad.wmnet -o es1019.eqiad.wmnet -t es2019.codfw.wmnet -c 25 -d databases.txt &> check.log\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ereal    15m15.129s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003euser    0m15.520s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003esys     0m10.376s\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ tail -n8 check.log\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e==== SUMMARY ====\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eAnalyzed 916 wikis\u003c\/span\u003e\n\u003cspan class=\"go\"\u003erev_text_id needed to be inverted for 122 wikis\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eOK: 604 wikis\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eNo revisions to check: 290 wikis\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eNo blob found for revision: 0 wikis\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eWith differences: 0 wikis\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eWith error getting revisions: 22 wikis\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eYou can find the results on \u003ctt class=\"remarkup-monospaced\"\u003eneodymium:\/home\/volans\/es2019.new\/\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2213436\" id=\"2213436\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2213436\" data-sigil=\"has-tooltip\" data-meta=\"0_194\"\u003e\u003cspan class=\"screen-only\"\u003eApr 18 2016, 8:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-18 08:23:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_192\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_193\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_42\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 283946 had a related patch set uploaded (by Volans):\u003cbr \/\u003e\nDBtools: add script to check external storage\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/283946\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/283946\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_199\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2213437\" id=\"2213437\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_198\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2213437\" data-sigil=\"has-tooltip\" data-meta=\"0_197\"\u003e\u003cspan class=\"screen-only\"\u003eApr 18 2016, 8:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-18 08:23:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_210\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_208\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2213475\" id=\"2213475\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_207\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2213475\" data-sigil=\"has-tooltip\" data-meta=\"0_206\"\u003e\u003cspan class=\"screen-only\"\u003eApr 18 2016, 8:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-18 08:49:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_204\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_205\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI think you are getting out of scope here- I did not ask to check mediawiki's data integrity- that is complex and we should stick to use mediawiki tools for that (there are like dozens of formats and exceptions). I am not saying that is not useful, I am saying that that is not easy and beyond infrastructure's goal. Otherwise, we would be reimplementing mediawiki's logic in bash.\u003c\/p\u003e\n\n\u003cp\u003eWhat we need is to check that replication was performed correctly, for that I suggested looking at the revision table and maybe get an approximate time band, then check the latest n records based on that. We could also look at the binary logs to check timestamp. You are probably hitting special cases such as flow tables stored on external storage and other oddities. Ids are calculated outside of mysql, so probably we cannot guarantee those happening serially either. But we can at least say "I am 100% sure that there are no differences in the records inserted in the last month".\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_221\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_219\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2213565\" id=\"2213565\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eVolans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2213565\" data-sigil=\"has-tooltip\" data-meta=\"0_217\"\u003e\u003cspan class=\"screen-only\"\u003eApr 18 2016, 9:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-18 09:11:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_215\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMy only problem is that I'm not sure that the blobs I'm checking were created on that period of time, that's the only thing I want to be sure of.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_232\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_230\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2213575\" id=\"2213575\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2213575\" data-sigil=\"has-tooltip\" data-meta=\"0_228\"\u003e\u003cspan class=\"screen-only\"\u003eApr 18 2016, 9:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-18 09:17:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_226\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eGive it a guess, then double the timeframe. E.g. estimate the number of edits per day, multiply by 30, then double it.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_243\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_241\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2225023\" id=\"2225023\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-check-square-o phui-timeline-icon\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eVolans\u003c\/a\u003e closed this task as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2225023\" data-sigil=\"has-tooltip\" data-meta=\"0_239\"\u003e\u003cspan class=\"screen-only\"\u003eApr 20 2016, 7:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-20 19:43:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_237\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_238\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_52\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ees2019 has been repoold after all executed checks passed.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_246\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2230524\" id=\"2230524\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-exclamation-circle phui-timeline-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003ejcrespo\u003c\/a\u003e reopened this task as \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230524\" data-sigil=\"has-tooltip\" data-meta=\"0_244\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 2:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 14:53:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_258\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_256\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2230546\" id=\"2230546\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_254\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003ejcrespo\u003c\/a\u003e edited projects, added \u003ca href=\"\/tag\/ops-codfw\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003eops-codfw\u003c\/a\u003e; removed \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003ePatch-For-Review\u003c\/a\u003e, \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003eDBA\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230546\" data-sigil=\"has-tooltip\" data-meta=\"0_253\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 3:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 15:03:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003ejcrespo\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Papaul\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003ePapaul\u003c\/a\u003e.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_251\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_252\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_60\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis server crashed again right now, potentially power related like the last time (there were no kernel logs last time, and we will probably will not found them either this time).\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Papaul\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Papaul\u003c\/span\u003e\u003c\/a\u003e could you please please check power related things, such as the power connection and the power supply. This is not a network issue, something makes it crash *really hard*. If you found nothing, please switch it on so we can very there are no logs. As a last resort, if there is nothing wrong with our installation, send it back to the provider as faulty.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_261\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2230549\" id=\"2230549\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_62\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Southparkfan\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003eSouthparkfan\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/1283528\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230549\" data-sigil=\"has-tooltip\" data-meta=\"0_259\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 3:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 15:03:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-grey\" data-sigil=\"transaction anchor-container\" data-meta=\"0_272\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_270\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2230550\" id=\"2230550\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-grey\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-down phui-timeline-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003ejcrespo\u003c\/a\u003e lowered the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230550\" data-sigil=\"has-tooltip\" data-meta=\"0_268\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 3:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 15:04:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_266\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNormal because we do not plan to use it any time soon (unlike last time).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_281\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2230558\" id=\"2230558\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230558\" data-sigil=\"has-tooltip\" data-meta=\"0_279\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 3:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 15:11:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_277\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 284903 had a related patch set uploaded (by Jcrespo):\u003cbr \/\u003e\nDepool es2019\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/284903\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/284903\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_290\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2230559\" id=\"2230559\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230559\" data-sigil=\"has-tooltip\" data-meta=\"0_288\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 3:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 15:11:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_286\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_70\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 284903 merged by Jcrespo:\u003cbr \/\u003e\nDepool es2019\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/284903\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/284903\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-black\" data-sigil=\"transaction anchor-container\" data-meta=\"0_294\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2230606\" id=\"2230606\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-black\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_292\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003ejcrespo\u003c\/a\u003e removed \u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\"\u003eVolans\u003c\/a\u003e as the assignee of this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230606\" data-sigil=\"has-tooltip\" data-meta=\"0_291\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 3:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 15:27:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003ejcrespo\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003ePatch-For-Review\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_297\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2230788\" id=\"2230788\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003ejcrespo\u003c\/a\u003e mentioned this in \u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003eUnknown Object (Task)\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2230788\" data-sigil=\"has-tooltip\" data-meta=\"0_295\"\u003e\u003cspan class=\"screen-only\"\u003eApr 22 2016, 4:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-22 16:16:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_306\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2ioid5yj3drfoxwvkj3a\/PHID-FILE-vnohjrunvspgobfqvfit\/alphanumeric_lato-white_P.png-_65558f-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Papaul\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2235352\" id=\"2235352\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Papaul\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003ePapaul\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2235352\" data-sigil=\"has-tooltip\" data-meta=\"0_304\"\u003e\u003cspan class=\"screen-only\"\u003eApr 25 2016, 3:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-25 15:19:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_302\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI did a hardware diagnostic,  everything looks good. The system is back up.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_318\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_316\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2235514\" id=\"2235514\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_315\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2235514\" data-sigil=\"has-tooltip\" data-meta=\"0_314\"\u003e\u003cspan class=\"screen-only\"\u003eApr 25 2016, 3:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-25 15:41:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_312\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_313\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Papaul\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Papaul\u003c\/span\u003e\u003c\/a\u003e Did you see anything strange regarding loose power connectors or any other thing that could lead to sudden server stop (silly things such as reset buttons in a bad position or potential short-circuits)? OR everything seemed correct? I am asking to see if the issue could be already fixed with your maintenance.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_327\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2ioid5yj3drfoxwvkj3a\/PHID-FILE-vnohjrunvspgobfqvfit\/alphanumeric_lato-white_P.png-_65558f-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Papaul\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2235613\" id=\"2235613\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Papaul\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003ePapaul\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2235613\" data-sigil=\"has-tooltip\" data-meta=\"0_325\"\u003e\u003cspan class=\"screen-only\"\u003eApr 25 2016, 3:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-25 15:57:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_323\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_324\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e I didn't notice anything.  I also make sure after the diagnostic i   unplugged the power, the mgmt and  data link for 5 minutes before putting everything back.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_338\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_336\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2235634\" id=\"2235634\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_335\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003ejcrespo\u003c\/a\u003e claimed this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2235634\" data-sigil=\"has-tooltip\" data-meta=\"0_334\"\u003e\u003cspan class=\"screen-only\"\u003eApr 25 2016, 4:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-04-25 16:06:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_332\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_333\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Papaul\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Papaul\u003c\/span\u003e\u003c\/a\u003e thank you very much for the time!\u003c\/p\u003e\n\n\u003cp\u003eI will restart now the service, but not put it back into production. I will keep meanwhile the ticket open for 2 months unless something happens.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ca27bo4fjnqillatgweg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6dn3grt7uh6sqcynyw6s","contextPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6dn3grt7uh6sqcynyw6s","contextPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6dn3grt7uh6sqcynyw6s","contextPHID":"PHID-TASK-jyj4xh2zodfc2usi26ud"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"phid":"PHID-XACT-TASK-mtpju4asj5lqfgj"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"phid":"PHID-XACT-TASK-du6vniishcyboxx"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-jozfja2wlmbztjk"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-hckdcvpe7t3zrbn"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bk6sgdzdhctxupi3vn5b"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-54xjtvb6c5vxcsr2izkm"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-dntuedaf2f635x6"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-54xjtvb6c5vxcsr2izkm"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-5iuvn4kjilwynldgts56"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-by4v3kvh3p5zcgh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-3uxsoebbb5julud"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-scfsdlx3t5nqm2h"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ijcsagz6gkenw2p"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-hl53hoyhrxagkna"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-7y7heg3ffsvb4rz"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-ilizaare5qehbap"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"phid":"PHID-XACT-TASK-tfm47u3ohjextlh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-heihjeaiasruuvneirzh"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6dn3grt7uh6sqcynyw6s"}},{"phid":"PHID-XACT-TASK-3vkwnco5vfxlqsz"},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-e7a5lnjpgncgdxknmdxb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-zbc3cysydx5hjrn"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-hh6nhwcz3jlyyrx"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"phid":"PHID-XACT-TASK-lzq5hdyfwqvzuhm"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-wc7jsvdv4ub6yvqxk77m"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6dn3grt7uh6sqcynyw6s"}},{"phid":"PHID-XACT-TASK-6bmrtj2uczsmaun"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-tiacfvnwzf7amsx"},{"hovercardSpec":{"objectPHID":"PHID-USER-6dn3grt7uh6sqcynyw6s"}},{"phid":"PHID-XACT-TASK-uoxrb5re37jkt6m"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-wwwbw7byalp5jsw"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uvkd76ehdoflc2u","anchor":"2143705"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-7ik22x7vxf7tjl7","anchor":"2143716"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mtpju4asj5lqfgj\/","ref":"T130702#2143718"},[],{"anchor":"2143718"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_90\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_91\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mtpju4asj5lqfgj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_92\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_93\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mtpju4asj5lqfgj","anchor":"2143718"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-du6vniishcyboxx\/","ref":"T130702#2143719"},[],{"anchor":"2143719"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_99\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_100\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-du6vniishcyboxx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_101\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_102\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-oodlh24armtttej","anchor":"2143719"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jozfja2wlmbztjk\/","ref":"T130702#2143737"},[],{"anchor":"2143737"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_108\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jozfja2wlmbztjk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_110\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-jozfja2wlmbztjk","anchor":"2143737"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hckdcvpe7t3zrbn\/","ref":"T130702#2143753"},[],{"anchor":"2143753"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_119\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hckdcvpe7t3zrbn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_121\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-hckdcvpe7t3zrbn\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-hckdcvpe7t3zrbn","anchor":"2143753"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-waqn66hwc5lw7fg","anchor":"2143754"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ogvvweoqlf7ncr7","anchor":"2143803"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dntuedaf2f635x6\/","ref":"T130702#2143818"},[],{"anchor":"2143818"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dntuedaf2f635x6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_139\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-uk5bixdnpa5hgyg","anchor":"2143818"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-rpsvyylef6c4gn7","anchor":"2209968"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-m7az3s6ux2bkpow","anchor":"2209986"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-by4v3kvh3p5zcgh\/","ref":"T130702#2212261"},[],{"anchor":"2212261"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_155\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-by4v3kvh3p5zcgh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_157\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-by4v3kvh3p5zcgh","anchor":"2212261"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3uxsoebbb5julud\/","ref":"T130702#2212826"},[],{"anchor":"2212826"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_166\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_167\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3uxsoebbb5julud\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_168\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3uxsoebbb5julud","anchor":"2212826"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-scfsdlx3t5nqm2h\/","ref":"T130702#2213435"},[],{"anchor":"2213435"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_177\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_178\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-scfsdlx3t5nqm2h\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_179\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_180\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-scfsdlx3t5nqm2h","anchor":"2213435"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ijcsagz6gkenw2p\/","ref":"T130702#2213436"},[],{"anchor":"2213436"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_188\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_189\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ijcsagz6gkenw2p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_190\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_191\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ijcsagz6gkenw2p","anchor":"2213436"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3csj43n62nepvxs","anchor":"2213437"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hl53hoyhrxagkna\/","ref":"T130702#2213475"},[],{"anchor":"2213475"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_200\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hl53hoyhrxagkna\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_202\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-hl53hoyhrxagkna","anchor":"2213475"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7y7heg3ffsvb4rz\/","ref":"T130702#2213565"},[],{"anchor":"2213565"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_211\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7y7heg3ffsvb4rz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_213\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_214\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-7y7heg3ffsvb4rz","anchor":"2213565"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ilizaare5qehbap\/","ref":"T130702#2213575"},[],{"anchor":"2213575"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_222\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ilizaare5qehbap\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_224\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ilizaare5qehbap","anchor":"2213575"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tfm47u3ohjextlh\/","ref":"T130702#2225023"},[],{"anchor":"2225023"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_233\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tfm47u3ohjextlh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_235\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-xzodzpkf3zpklzc","anchor":"2225023"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pyyjtswrhuvayhj","anchor":"2230524"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3vkwnco5vfxlqsz\/","ref":"T130702#2230546"},[],{"anchor":"2230546"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_247\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3vkwnco5vfxlqsz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_249\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_250\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-72gzfebm3trnucm","anchor":"2230546"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-c4hao7gzfmta4sf","anchor":"2230549"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zbc3cysydx5hjrn\/","ref":"T130702#2230550"},[],{"anchor":"2230550"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_262\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_263\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zbc3cysydx5hjrn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_264\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-lf2e2aa22oghotl","anchor":"2230550"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hh6nhwcz3jlyyrx\/","ref":"T130702#2230558"},[],{"anchor":"2230558"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_273\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hh6nhwcz3jlyyrx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_275\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-hh6nhwcz3jlyyrx","anchor":"2230558"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lzq5hdyfwqvzuhm\/","ref":"T130702#2230559"},[],{"anchor":"2230559"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_282\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lzq5hdyfwqvzuhm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_284\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4ya4pobsenheehk","anchor":"2230559"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-rt6lx7jxa5bjbs4","anchor":"2230606"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lqbqjfd7czqtq73","anchor":"2230788"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6bmrtj2uczsmaun\/","ref":"T130702#2235352"},[],{"anchor":"2235352"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_298\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_299\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6bmrtj2uczsmaun\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_300\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6bmrtj2uczsmaun","anchor":"2235352"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tiacfvnwzf7amsx\/","ref":"T130702#2235514"},[],{"anchor":"2235514"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_307\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tiacfvnwzf7amsx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_309\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-tiacfvnwzf7amsx\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tiacfvnwzf7amsx","anchor":"2235514"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uoxrb5re37jkt6m\/","ref":"T130702#2235613"},[],{"anchor":"2235613"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_319\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uoxrb5re37jkt6m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_321\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_322\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uoxrb5re37jkt6m","anchor":"2235613"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wwwbw7byalp5jsw\/","ref":"T130702#2235634"},[],{"anchor":"2235634"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_328\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_329\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wwwbw7byalp5jsw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_330\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_331\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ap6ht6f4om6zukh","anchor":"2235634"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}