for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_102\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2912830\" id=\"2912830\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2912830\" data-sigil=\"has-tooltip\" data-meta=\"0_100\"\u003e\u003cspan class=\"screen-only\"\u003eJan 3 2017, 3:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-03 15:14:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2912843\" id=\"2912843\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_104\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_26\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/1646557\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2912843\" data-sigil=\"has-tooltip\" data-meta=\"0_103\"\u003e\u003cspan class=\"screen-only\"\u003eJan 3 2017, 3:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-03 15:14:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2912844\" id=\"2912844\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2912844\" data-sigil=\"has-tooltip\" data-meta=\"0_106\"\u003e\u003cspan class=\"screen-only\"\u003eJan 3 2017, 3:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-03 15:14:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_119\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2916083\" id=\"2916083\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2916083\" data-sigil=\"has-tooltip\" data-meta=\"0_115\"\u003e\u003cspan class=\"screen-only\"\u003eJan 4 2017, 9:54 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-04 09:54:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_113\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_30\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChecking the latest version of percona toolkit (v2.2.20 released 6th Dec 2016) it contains Jaime's patch (actually since 2.2.19):\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFixed bug 1485195: pt-table-checksum should force replica table charset to utf8 Edit (Thanks Jaime Crespo)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAnd it also includes a new feature that might be useful for us:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFeature   1610385: Recheck the list of slaves while OSC runs (Thanks Dani\u00ebl van Eeden & Mikhail Izioumtchenko)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_130\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_131\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_132\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_133\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2919046\" id=\"2919046\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2919046\" data-sigil=\"has-tooltip\" data-meta=\"0_128\"\u003e\u003cspan class=\"screen-only\"\u003eJan 5 2017, 10:29 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-05 10:29:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_126\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_32\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have been doing a few tests with the \u003ctt class=\"remarkup-monospaced\"\u003e--recursion-method=dsn\u003c\/tt\u003e  to only check a selected number of hosts:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eMariaDB [percona]> show create table dsns\\G\n*************************** 1. row ***************************\n       Table: dsns\nCreate Table: CREATE TABLE `dsns` (\n  `id` int(11) NOT NULL AUTO_INCREMENT,\n  `parent_id` int(11) DEFAULT NULL,\n  `dsn` varchar(255) NOT NULL,\n  PRIMARY KEY (`id`)\n) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eOnly one entry there:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eMariaDB [percona]> select * from dsns;\n+----+-----------+-----------------------------------------------------+\n| id | parent_id | dsn                                                 |\n+----+-----------+-----------------------------------------------------+\n|  1 |      NULL | h=192.168.56.12,u=checksum_user,p=checksum_password |\n+----+-----------+-----------------------------------------------------+\n1 row in set (0.00 sec)\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eBut the master has two slaves:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eMariaDB [percona]> show slave hosts;\n+-----------+---------------+------+-----------+\n| Server_id | Host          | Port | Master_id |\n+-----------+---------------+------+-----------+\n|         3 | 192.168.56.13 | 3306 |         1 |\n|         2 | 192.168.56.12 | 3306 |         1 |\n+-----------+---------------+------+-----------+\n2 rows in set (0.00 sec)\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eIt works well and can be a good option for the future inventory as in: only check this slaves\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clone:~\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --replicate=percona.checksums --replicate-check-only --ignore-databases mysql h=localhost,u=checksum_user,p=checksum_password --recursion-method=dsn=h=localhost,D=percona,t=dsns\nDifferences on slave1\nTABLE CHUNK CNT_DIFF CRC_DIFF CHUNK_INDEX LOWER_BOUNDARY UPPER_BOUNDARY\ndummy.dummy 1 -5 1\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd using the normal \u003ctt class=\"remarkup-monospaced\"\u003ehosts\u003c\/tt\u003e method it shows differences in both slaves:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clone:~\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --replicate=percona.checksums --replicate-check-only --ignore-databases mysql h=localhost,u=checksum_user,p=checksum_password\nDifferences on slave1\nTABLE CHUNK CNT_DIFF CRC_DIFF CHUNK_INDEX LOWER_BOUNDARY UPPER_BOUNDARY\ndummy.dummy 1 -5 1\n\nDifferences on slave2\nTABLE CHUNK CNT_DIFF CRC_DIFF CHUNK_INDEX LOWER_BOUNDARY UPPER_BOUNDARY\ndummy.dummy 1 -5 1\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eIf we delay a slave and update the DSN to only check the non delayed slave it looks good:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clone:~\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --replicate=percona.checksums --ignore-databases mysql --recursion-method=dsn=h=localhost,D=percona,t=dsns\nReplica debian is stopped.  Waiting.\n\n\nMariaDB [percona]> update dsns set dsn="h=192.168.56.13,u=checksum_user,p=checksum_password" ;\nQuery OK, 1 row affected (0.00 sec)\nRows matched: 1  Changed: 1  Warnings: 0\n\nroot@clone:~\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --replicate=percona.checksums --ignore-databases mysql --recursion-method=dsn=h=localhost,D=percona,t=dsns\n            TS ERRORS  DIFFS     ROWS  CHUNKS SKIPPED    TIME TABLE\n01-05T05:26:18      0      1       10       1       0   0.023 dummy.dummy\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe \u003ctt class=\"remarkup-monospaced\"\u003e--check-slave-lag\u003c\/tt\u003e option to specify which slaves we want to specifically check for lag also looks good and useful for us.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clone:~\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --replicate=percona.checksums --ignore-databases mysql h=localhost,u=checksum_user,p=checksum_password --check-slave-lag 192.168.56.12\nReplica slave1 is stopped.  Waiting.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eNow trying only checking the other slave which isn't delayed or stopped while the other replica is stopped:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@clone:~\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --replicate=percona.checksums --ignore-databases mysql h=localhost,u=checksum_user,p=checksum_password --check-slave-lag 192.168.56.13\n            TS ERRORS  DIFFS     ROWS  CHUNKS SKIPPED    TIME TABLE\n01-05T05:23:58      0      1       10       1       0   0.032 dummy.dummy\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2928739\" id=\"2928739\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a parent task: \u003ca href=\"\/T134476\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eT134476: Decommission old coredb machines (<=db1050)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2928739\" data-sigil=\"has-tooltip\" data-meta=\"0_135\"\u003e\u003cspan class=\"screen-only\"\u003eJan 9 2017, 7:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-09 19:53:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_140\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2934674\" id=\"2934674\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_139\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eTriage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eBacklog\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eDBA\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2934674\" data-sigil=\"has-tooltip\" data-meta=\"0_138\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2017, 11:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-11 23:21:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2992332\" id=\"2992332\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e mentioned this in \u003ca href=\"\/T156844\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eT156844: Decommission old dbstore hosts (db1046, db1047)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2992332\" data-sigil=\"has-tooltip\" data-meta=\"0_141\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 2 2017, 7:36 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-02 07:36:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_152\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_153\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_154\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3024248\" id=\"3024248\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_151\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3024248\" data-sigil=\"has-tooltip\" data-meta=\"0_150\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2017, 8:45 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-14 08:45:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_148\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_42\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e I would like to run a first iteration of pt-table-checksum on m3 maybe, to see how it goes.\u003cbr \/\u003e\nI would be using the dsns side table as I stated above to see how it goes.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_167\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_165\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3024425\" id=\"3024425\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3024425\" data-sigil=\"has-tooltip\" data-meta=\"0_163\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2017, 10:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-14 10:12:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_161\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_44\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOk, but do not create percona databases on production, it is a source of garbage and security issues. I used separate tables per database called __wmf_checksums in the past, you can reuse that, or just use your own thing, but try to avoid new database creation.\u003c\/p\u003e\n\n\u003cp\u003eI already run it on misc servers in the past, BTW, but it will be interesting after all crashes.\u003c\/p\u003e\n\n\u003cp\u003eAlso Important! give a look at: \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T104310\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T104310\u003c\/span\u003e\u003c\/a\u003e Otherwise, replication will break on labs hosts.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_176\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3024428\" id=\"3024428\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3024428\" data-sigil=\"has-tooltip\" data-meta=\"0_174\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2017, 10:15 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-14 10:15:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_172\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3024425\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3024425\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eOk, but do not create percona databases on production, it is a source of garbage and security issues. I used separate tables per database called __wmf_checksums in the past, you can reuse that, or just use your own thing, but try to avoid new database creation.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYeah, no worries. I will place that new table somewhere else (maybe ops).\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI already run it on misc servers in the past, BTW, but it will be interesting after all crashes.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eDo you have that command handy? Maybe it will be interesting to run one as similar as possible as the one you ran.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eAlso Important! give a look at: \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T104310\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_7\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T104310\u003c\/span\u003e\u003c\/a\u003e Otherwise, replication will break on labs hosts.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eYes, thanks, will carefully review that\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_193\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_189\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_190\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_191\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_192\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3025051\" id=\"3025051\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_188\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3025051\" data-sigil=\"has-tooltip\" data-meta=\"0_187\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2017, 1:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-14 13:58:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_185\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe will need to also make sure we do not include hosts running RBR (ie: sanitarium masters):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ept-table-checksum requires statement-based replication, and it sets binlog_format=STATEMENT on the master, but due to a MySQL limitation replicas do not honor this change. Therefore, checksums will not replicate past any replicas using row-based replication that are masters for further replicas.\n\nThe tool automatically checks the binlog_format on all servers. See --[no]check-binlog-format .\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_197\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3025449\" id=\"3025449\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003ejcrespo\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3025449\" data-sigil=\"has-tooltip\" data-meta=\"0_194\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2017, 3:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-14 15:40:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_196\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003ejcrespo\u003c\/a\u003e edited parent tasks, added: \u003ca href=\"\/T148078\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eT148078: Decommission db1015, db1035, db1044 and db1038\u003c\/a\u003e; removed: \u003ca href=\"\/T134476\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003eT134476: Decommission old coredb machines (<=db1050)\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_210\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_206\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_207\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_208\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028719\" id=\"3028719\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_205\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028719\" data-sigil=\"has-tooltip\" data-meta=\"0_204\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 10:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 10:09:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_202\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_54\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI assume the following is going to happen in all the hosts as we have MIXED everywhere (but some specific cases like the sanitarium2 masters):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@neodymium:~# pt-table-checksum --replicate=phabricator_file.__wmf_checksums --noreplicate-check --check-slave-lag db1048.eqiad.wmnet --max-lag 2 -F \/home\/marostegui\/.my.cnf  --databases phabricator_file h=db1043.eqiad.wmnet,u=root --recursion-method=dsn=h=db1043.eqiad.wmnet,D=phabricator_file,t=__dsns\n\nReplica db1048 has binlog_format MIXED which could cause pt-table-checksum to break replication.  Please read "Replicas using row-based replication" in the LIMITATIONS section of the tool's documentation.  If you understand the risks, specify --no-check-binlog-format to disable this check.\nReplica db2012 has binlog_format MIXED which could cause pt-table-checksum to break replication.  Please read "Replicas using row-based replication" in the LIMITATIONS section of the tool's documentation.  If you understand the risks, specify --no-check-binlog-format to disable this check.\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_221\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_219\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028736\" id=\"3028736\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028736\" data-sigil=\"has-tooltip\" data-meta=\"0_217\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 10:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 10:26:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_215\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_56\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eUse "--no-check-binlog-format"- pt-t-c forces binlog format already if using super for itself, it should only cause issues for multi-level slaves, but you can check later on the binlog that queries are done in STATEMENT format as expected.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_232\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_230\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028741\" id=\"3028741\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028741\" data-sigil=\"has-tooltip\" data-meta=\"0_228\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 10:28 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 10:28:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_226\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_58\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAlso, let's centralize the dsn tables on tendril or any other central place- so we do not have garbage tables in the future scattered all around.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_244\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_242\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028742\" id=\"3028742\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#3028742\" data-sigil=\"has-tooltip\" data-meta=\"0_240\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 10:29 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 10:29:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_238\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_60\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAre you sure also you are using an updated version of pt-table-checksum, one without the binary bug? why --no-replicate-check ?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_257\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_253\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_254\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_255\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028759\" id=\"3028759\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_252\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028759\" data-sigil=\"has-tooltip\" data-meta=\"0_251\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 10:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 10:46:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_249\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_250\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028736\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028736\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eUse "--no-check-binlog-format"- pt-t-c forces binlog format already if using super for itself, it should only cause issues for multi-level slaves, but you can check later on the binlog that queries are done in STATEMENT format as expected.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThanks, I wasn't 100% aware of the implications of it.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028741\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028741\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAlso, let's centralize the dsn tables on tendril or any other central place- so we do not have garbage tables in the future scattered all around.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThat makes sense, however we'd need to truncate the table after using it as it will be used to check specific slaves from different shards.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028742\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028742\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAre you sure also you are using an updated version of pt-table-checksum, one without the binary bug? why --no-replicate-check ?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eGood point: pt-table-checksum 2.2.20\u003cbr \/\u003e\nThe --no-replicate-check was just a test for this first run on the phab master. It will be removed (as the default for it is to run)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_268\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_266\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028802\" id=\"3028802\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028802\" data-sigil=\"has-tooltip\" data-meta=\"0_264\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 11:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 11:03:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_262\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_263\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_64\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eGood point: pt-table-checksum 2.2.20\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI do not know when that was fixed- \u003ctt class=\"remarkup-monospaced\"\u003egrep -A 15 ' CREATE TABLE checksums' $(which pt-table-checksum)\u003c\/tt\u003e should force the table or the char fields to be utf8, or execution will fail on our production (getting stuck forever).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_279\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_277\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028813\" id=\"3028813\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028813\" data-sigil=\"has-tooltip\" data-meta=\"0_275\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 11:06 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 11:06:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_273\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_66\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028759\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028759\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThat makes sense, however we'd need to truncate the table after using it as it will be used to check specific slaves from different shards.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eLet's create a dsns database on tendril, we can create one table per shard so we do not have to work twice next time! :-)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_292\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_288\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_290\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028822\" id=\"3028822\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028822\" data-sigil=\"has-tooltip\" data-meta=\"0_286\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 11:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 11:09:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_284\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_68\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028802\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028802\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eGood point: pt-table-checksum 2.2.20\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI do not know when that was fixed- \u003ctt class=\"remarkup-monospaced\"\u003egrep -A 15 ' CREATE TABLE checksums' $(which pt-table-checksum)\u003c\/tt\u003e should force the table or the char fields to be utf8, or execution will fail on our production (getting stuck forever).\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIt is fixed on 2.2.19 and 2.2.20 is the one I have in my home directory (but we should indeed upgrade the system one):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@neodymium:\/home\/marostegui\/percona-toolkit-2.2.20\/bin# grep -A 15 ' CREATE TABLE checksums' pt-table-checksum\n  CREATE TABLE checksums (\n     db             CHAR(64)     NOT NULL,\n     tbl            CHAR(64)     NOT NULL,\n     chunk          INT          NOT NULL,\n     chunk_time     FLOAT            NULL,\n     chunk_index    VARCHAR(200)     NULL,\n     lower_boundary TEXT             NULL,\n     upper_boundary TEXT             NULL,\n     this_crc       CHAR(40)     NOT NULL,\n     this_cnt       INT          NOT NULL,\n     master_crc     CHAR(40)         NULL,\n     master_cnt     INT              NULL,\n     ts             TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n     PRIMARY KEY (db, tbl, chunk),\n     INDEX ts_db_tbl (ts, db, tbl)\n  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028813\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028813\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T154485#3028759\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485#3028759\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-orange\"\u003e\u003c\/span\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThat makes sense, however we'd need to truncate the table after using it as it will be used to check specific slaves from different shards.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eLet's create a dsns database on tendril, we can create one table per shard so we do not have to work twice next time! :-)\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eSounds good. I will take care of that\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_301\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028895\" id=\"3028895\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003eStashbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028895\" data-sigil=\"has-tooltip\" data-meta=\"0_299\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 11:38 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 11:38:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_297\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AVpBkLgPlCyyDMEPvLj9\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2017-02-15T11:38:07Z] <marostegui> Running pt-table-checksum on db1043 (m3 - phabricator master) - \u003ca href=\"\/T154485\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_314\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_310\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_312\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_313\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3028898\" id=\"3028898\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_309\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028898\" data-sigil=\"has-tooltip\" data-meta=\"0_308\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 11:43 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 11:43:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_306\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_72\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFor the record I have created the dsns tables on tendril (and the first test with pt-table-checksum on m3 is using it).\u003cbr \/\u003e\nThe only one that has data so far is dsns_m3\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eMariaDB TENDRIL localhost dsns > show tables;\n+----------------+\n| Tables_in_dsns |\n+----------------+\n| dsns_m3        |\n| dsns_s1        |\n| dsns_s2        |\n| dsns_s3        |\n| dsns_s4        |\n| dsns_s5        |\n| dsns_s6        |\n| dsns_s7        |\n| dsns_s8        |\n+----------------+\n9 rows in set (0.00 sec)\n\nMariaDB TENDRIL localhost dsns > show create table dsns_m3;\n+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| Table   | Create Table                                                                                                                                                                                                                                              |\n+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| dsns_m3 | CREATE TABLE `dsns_m3` (\n  `id` int(11) NOT NULL AUTO_INCREMENT,\n  `parent_id` int(11) DEFAULT NULL,\n  `dsn` varchar(255) COLLATE utf8mb4_bin NOT NULL,\n  PRIMARY KEY (`id`)\n) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin |\n+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n1 row in set (0.00 sec)\n\nMariaDB TENDRIL localhost dsns > select * from dsns_m3;\n+----+-----------+-----------------------------+\n| id | parent_id | dsn                         |\n+----+-----------+-----------------------------+\n|  1 |      NULL | h=db1048.eqiad.wmnet,u=root |\n|  2 |      NULL | h=db2012.codfw.wmnet,u=root |\n+----+-----------+-----------------------------+\n2 rows in set (0.00 sec)\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_318\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3028952\" id=\"3028952\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-trophy phui-timeline-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view sprite-tokens tokens-heart-1 phui-timeline-token\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003ejcrespo\u003c\/a\u003e awarded a token.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3028952\" data-sigil=\"has-tooltip\" data-meta=\"0_315\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 12:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 12:08:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_331\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_329\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3030925\" id=\"3030925\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3030925\" data-sigil=\"has-tooltip\" data-meta=\"0_325\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 15 2017, 9:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-15 21:17:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_323\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_324\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe first test with phabricator_file database and generated a peak of 500 seconds lag on db1048 and db2012 while checksuming the biggest table of the database: file_storageblog (26GB).\u003cbr \/\u003e\nI didn't specified any \u003ctt class=\"remarkup-monospaced\"\u003echunk-size\u003c\/tt\u003e or \u003ctt class=\"remarkup-monospaced\"\u003echunk-time\u003c\/tt\u003e to see how the tool would adjust (that means a chunk-size=1000 and chunk-time=0.5 by default), but it didn't do it so well (as I said, 500 seconds lag was generated). The \u003ctt class=\"remarkup-monospaced\"\u003e--max-lag\u003c\/tt\u003e was set to "2"\u003cbr \/\u003e\nThis is what was run:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e.\/pt-table-checksum --replicate-check-only --no-check-binlog-format --replicate=phabricator_file.__wmf_checksums --check-slave-lag db1048.eqiad.wmnet,db2012.codfw.wmnet --max-lag 1 -F \/home\/marostegui\/.my.cnf  --databases phabricator_file h=db1043.eqiad.wmnet,u=root --recursion-method=dsn=h=db1011.eqiad.wmnet,D=dsns,t=dsns_m3\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe biggest tables took 1744 seconds to be checksummed\u003c\/p\u003e\n\n\u003cp\u003eFor the next iteration I have specified a chunk-time of 0.1 which makes the tool slower but safer (we even get a warning about how slow it is).\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@neodymium:\/home\/marostegui\/percona-toolkit-2.2.20\/bin# .\/pt-table-checksum --no-check-binlog-format --replicate=phabricator_file.__wmf_checksums --check-slave-lag db1048.eqiad.wmnet,db2012.codfw.wmnet --max-lag 1 --chunk-time 0.1 -F \/home\/marostegui\/.my.cnf  --databases phabricator_file h=db1043.eqiad.wmnet,u=root --recursion-method=dsn=h=db1011.eqiad.wmnet,D=dsns,t=dsns_m3\n\n<snip>\n\n02-15T13:48:41 Checksum queries for table phabricator_file.file_storageblob are executing very slowly.  --chunk-size has been automatically reduced to 1.  Check that the server is not being overloaded, or increase --chunk-time.  The last chunk, number 92 of table phabricator_file.file_storageblob, selected 1 rows and took 0.123 seconds to execute.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eDespite this, the replicas got around 40 seconds delay peak. \u003cbr \/\u003e\nThis time the table took 1886 seconds to be checksummed and it was a way more gentle to the replicas.\u003c\/p\u003e\n\n\u003cp\u003eAnother run I did was to leave the \u003ctt class=\"remarkup-monospaced\"\u003echunk-time\u003c\/tt\u003e to be the default one (0.5) and play with the chunk-size instead of the default 1000 rows to 500 rows.\u003cbr \/\u003e\nIt is a lot slower, but there was no lag reported by pt-table-checksum\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e.\/pt-table-checksum --no-check-binlog-format --replicate=phabricator_file.__wmf_checksums --check-slave-lag db1048.eqiad.wmnet,db2012.codfw.wmnet --max-lag 1 --chunk-size 500 -F \/home\/marostegui\/.my.cnf  --databases phabricator_file h=db1043.eqiad.wmnet,u=root --recursion-method=dsn=h=db1011.eqiad.wmnet,D=dsns,t=dsns_m3\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThis time the whole check took around 40 minutes and 20 minutes to checksum a 26G table. \u003cbr \/\u003e\nAs I said, the tool didn't report any lag this time, but I left a script running to check whether that was true and it is mostly true, the max lag I was able to capture from a direct replica (within the same DC) was 19 seconds:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 13\nSeconds_Behind_Master: 11\nSeconds_Behind_Master: 13\nSeconds_Behind_Master: 10\nSeconds_Behind_Master: 16\nSeconds_Behind_Master: 19\nSeconds_Behind_Master: 13\nSeconds_Behind_Master: 10\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe time between each check is 5 seconds. So it was delayed for around 40 seconds but no more than 20 seconds at the time, which seems pretty good to me. It is true that those servers do not have any reads, so maybe we need to decrease the chunk-size a bit more for production.\u003c\/p\u003e\n\n\u003cp\u003eThe last test was \u003ctt class=\"remarkup-monospaced\"\u003echunk-size 200\u003c\/tt\u003e:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e.\/pt-table-checksum --no-check-binlog-format --replicate=phabricator_file.__wmf_checksums --check-slave-lag db1048.eqiad.wmnet,db2012.codfw.wmnet --max-lag 1 --chunk-size 200 -F \/home\/marostegui\/.my.cnf  --databases phabricator_file h=db1043.eqiad.wmnet,u=root --recursion-method=dsn=h=db1011.eqiad.wmnet,D=dsns,t=dsns_m3\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe whole check took 1 hour. But it generated less lag even, these are the two lag spikes it got during the different table checks:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 1\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 5\nSeconds_Behind_Master: 2\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 1\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 1\nSeconds_Behind_Master: 1\nSeconds_Behind_Master: 1\nSeconds_Behind_Master: 2\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 5\nSeconds_Behind_Master: 7\nSeconds_Behind_Master: 10\nSeconds_Behind_Master: 8\nSeconds_Behind_Master: 7\nSeconds_Behind_Master: 6\nSeconds_Behind_Master: 8\nSeconds_Behind_Master: 6\nSeconds_Behind_Master: 3\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 3\nSeconds_Behind_Master: 8\nSeconds_Behind_Master: 13\nSeconds_Behind_Master: 12\nSeconds_Behind_Master: 2\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe last ran with \u003ctt class=\"remarkup-monospaced\"\u003echunk-size\u003c\/tt\u003e took 1:50h to finish the whole dataset. The lag generated was a bit less...\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 1\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 5\nSeconds_Behind_Master: 2\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 1\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnother batch\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 2\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 5\nSeconds_Behind_Master: 7\nSeconds_Behind_Master: 10\nSeconds_Behind_Master: 8\nSeconds_Behind_Master: 7\nSeconds_Behind_Master: 6\nSeconds_Behind_Master: 8\nSeconds_Behind_Master: 6\nSeconds_Behind_Master: 3\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnother batch\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 3\nSeconds_Behind_Master: 8\nSeconds_Behind_Master: 13\nSeconds_Behind_Master: 12\nSeconds_Behind_Master: 2\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnother one\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSeconds_Behind_Master: 2\nSeconds_Behind_Master: 3\nSeconds_Behind_Master: 7\nSeconds_Behind_Master: 10\nSeconds_Behind_Master: 5\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo the chunk-size looks safer but it will take quite long (which is fine, I rather not generate lag in production)\u003c\/p\u003e\n\n\u003cp\u003eAs a side note: the \u003ctt class=\"remarkup-monospaced\"\u003e--ignore-tables\u003c\/tt\u003e works fine\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_344\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_340\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_342\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3032453\" id=\"3032453\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3032453\" data-sigil=\"has-tooltip\" data-meta=\"0_338\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 16 2017, 9:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-16 09:19:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_336\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003edbstore1001 struggled to execute the pt-table-checksum query:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eREPLACE INTO `phabricator_file`.`__wmf_checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT 'phabricator_file', 'file_storageblob', '1', NULL, NULL, NULL, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `id`, CRC32(`data`), `datecreated`, `datemodified`)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `phabricator_file`.`file_storageblob` \/*checksum table*\/\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eIt made the server kind of hang (didn't crash at all and when I killed that replace that it went fine straightaway), this was the status:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@DBSTORE[(none)]> show processlist;\n+----------+-----------------+------------------+--------------------+---------+---------+----------------------------------+------------------------------------------------------------------------------------------------------+----------+\n| Id       | User            | Host             | db                 | Command | Time    | State                            | Info                                                                                                 | Progress |\n+----------+-----------------+------------------+--------------------+---------+---------+----------------------------------+------------------------------------------------------------------------------------------------------+----------+\n|  7927022 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927024 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927026 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927028 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927030 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927032 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927034 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927036 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927038 | system user     |                  | NULL               | Connect | 1354892 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927050 | event_scheduler | localhost        | NULL               | Daemon  |       1 | Waiting for next activation      | NULL                                                                                                 |    0.000 |\n| 10107012 | watchdog        | 10.64.0.15:48258 | mysql              | Sleep   |       6 |                                  | NULL                                                                                                 |    0.000 |\n| 10138755 | watchdog        | 10.64.0.15:51997 | information_schema | Query   |     209 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10142021 | watchdog        | 10.64.0.15:34146 | information_schema | Query   |     220 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_VARIABLES`                                     |    0.000 |\n| 10142049 | watchdog        | 10.64.0.15:34909 | information_schema | Sleep   |      16 |                                  | NULL                                                                                                 |    0.000 |\n| 10187049 | system user     |                  | phabricator_file   | Connect |   77588 | Sending data                     | REPLACE INTO `phabricator_file`.`__wmf_checksums` (db, tbl, chunk, chunk_index, lower_boundary, uppe |    0.000 |\n| 10187055 | root            | localhost        | ops                | Connect |     231 | Killing slave                    | stop slave "m3" sql_thread                                                                           |    0.000 |\n| 10187058 | watchdog        | 10.64.0.15:51748 | information_schema | Query   |     230 | init                             | SHOW ALL SLAVES STATUS                                                                               |    0.000 |\n| 10187059 | nagios          | localhost        | NULL               | Query   |     230 | init                             | show slave status                                                                                    |    0.000 |\n| 10187060 | nagios          | localhost        | NULL               | Query   |     230 | init                             | show slave status                                                                                    |    0.000 |\n| 10187061 | nagios          | localhost        | NULL               | Query   |     230 | init                             | show slave status                                                                                    |    0.000 |\n| 10187062 | nagios          | localhost        | NULL               | Query   |     230 | init                             | show slave status                                                                                    |    0.000 |\n| 10187063 | nagios          | localhost        | NULL               | Query   |     228 | init                             | show slave status                                                                                    |    0.000 |\n| 10187064 | nagios          | localhost        | NULL               | Query   |     227 | init                             | show slave status                                                                                    |    0.000 |\n| 10187068 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187069 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187070 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187071 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187072 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187073 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187074 | nagios          | localhost        | NULL               | Query   |     213 | init                             | show slave status                                                                                    |    0.000 |\n| 10187075 | nagios          | localhost        | NULL               | Query   |     212 | init                             | show slave status                                                                                    |    0.000 |\n| 10187076 | nagios          | localhost        | NULL               | Query   |     212 | init                             | show slave status                                                                                    |    0.000 |\n| 10187077 | nagios          | localhost        | NULL               | Query   |     212 | init                             | show slave status                                                                                    |    0.000 |\n| 10187079 | nagios          | localhost        | NULL               | Query   |     208 | init                             | show slave status                                                                                    |    0.000 |\n| 10187080 | nagios          | localhost        | NULL               | Query   |     208 | init                             | show slave status                                                                                    |    0.000 |\n| 10187081 | nagios          | localhost        | NULL               | Query   |     208 | init                             | show slave status                                                                                    |    0.000 |\n| 10187082 | nagios          | localhost        | NULL               | Query   |     207 | init                             | show slave status                                                                                    |    0.000 |\n| 10187083 | nagios          | localhost        | NULL               | Query   |     207 | init                             | show slave status                                                                                    |    0.000 |\n| 10187084 | nagios          | localhost        | NULL               | Query   |     207 | init                             | show slave status                                                                                    |    0.000 |\n| 10187086 | nagios          | localhost        | NULL               | Query   |     205 | init                             | show slave status                                                                                    |    0.000 |\n| 10187087 | nagios          | localhost        | NULL               | Query   |     204 | init                             | show slave status                                                                                    |    0.000 |\n| 10187088 | nagios          | localhost        | NULL               | Query   |     204 | init                             | show slave status                                                                                    |    0.000 |\n| 10187090 | nagios          | localhost        | NULL               | Query   |     201 | init                             | show slave status                                                                                    |    0.000 |\n| 10187091 | nagios          | localhost        | NULL               | Query   |     201 | init                             | show slave status                                                                                    |    0.000 |\n| 10187092 | prometheus      | localhost        | NULL               | Query   |     199 | Filling schema table             | SHOW GLOBAL STATUS                                                                                   |    0.000 |\n| 10187093 | nagios          | localhost        | NULL               | Query   |     198 | init                             | show slave status                                                                                    |    0.000 |\n| 10187094 | nagios          | localhost        | NULL               | Query   |     198 | init                             | show slave status                                                                                    |    0.000 |\n| 10187095 | nagios          | localhost        | NULL               | Query   |     197 | init                             | show slave status                                                                                    |    0.000 |\n| 10187096 | nagios          | localhost        | NULL               | Query   |     197 | init                             | show slave status                                                                                    |    0.000 |\n| 10187098 | nagios          | localhost        | NULL               | Query   |     195 | init                             | show slave status                                                                                    |    0.000 |\n| 10187099 | nagios          | localhost        | NULL               | Query   |     195 | init                             | show slave status                                                                                    |    0.000 |\n| 10187100 | nagios          | localhost        | NULL               | Query   |     195 | init                             | show slave status                                                                                    |    0.000 |\n| 10187101 | nagios          | localhost        | NULL               | Query   |     195 | init                             | show slave status                                                                                    |    0.000 |\n| 10187102 | nagios          | localhost        | NULL               | Query   |     195 | init                             | show slave status                                                                                    |    0.000 |\n| 10187103 | nagios          | localhost        | NULL               | Query   |     192 | init                             | show slave status                                                                                    |    0.000 |\n| 10187105 | nagios          | localhost        | NULL               | Query   |     190 | init                             | show slave status                                                                                    |    0.000 |\n| 10187106 | prometheus      | localhost        | NULL               | Query   |     189 | Filling schema table             | SHOW GLOBAL STATUS                                                                                   |    0.000 |\n| 10187107 | nagios          | localhost        | NULL               | Query   |     189 | init                             | show slave status                                                                                    |    0.000 |\n| 10187108 | nagios          | localhost        | NULL               | Query   |     188 | init                             | show slave status                                                                                    |    0.000 |\n| 10187109 | nagios          | localhost        | NULL               | Query   |     188 | init                             | show slave status                                                                                    |    0.000 |\n| 10187110 | nagios          | localhost        | NULL               | Query   |     188 | init                             | show slave status                                                                                    |    0.000 |\n| 10187111 | nagios          | localhost        | NULL               | Query   |     188 | init                             | show slave status                                                                                    |    0.000 |\n| 10187113 | nagios          | localhost        | NULL               | Query   |     186 | init                             | show slave status                                                                                    |    0.000 |\n| 10187114 | nagios          | localhost        | NULL               | Query   |     186 | init                             | show slave status                                                                                    |    0.000 |\n| 10187115 | watchdog        | 10.64.0.15:53215 | information_schema | Query   |     160 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_VARIABLES`                                     |    0.000 |\n| 10187116 | nagios          | localhost        | NULL               | Query   |     185 | init                             | show slave status                                                                                    |    0.000 |\n| 10187117 | nagios          | localhost        | NULL               | Query   |     184 | init                             | show slave status                                                                                    |    0.000 |\n| 10187118 | nagios          | localhost        | NULL               | Query   |     184 | init                             | show slave status                                                                                    |    0.000 |\n| 10187119 | nagios          | localhost        | NULL               | Query   |     183 | init                             | show slave status                                                                                    |    0.000 |\n| 10187120 | nagios          | localhost        | NULL               | Query   |     182 | init                             | show slave status                                                                                    |    0.000 |\n| 10187121 | nagios          | localhost        | NULL               | Query   |     182 | init                             | show slave status                                                                                    |    0.000 |\n| 10187122 | nagios          | localhost        | NULL               | Query   |     182 | init                             | show slave status                                                                                    |    0.000 |\n| 10187123 | nagios          | localhost        | NULL               | Query   |     182 | init                             | show slave status                                                                                    |    0.000 |\n| 10187125 | nagios          | localhost        | NULL               | Query   |     180 | init                             | show slave status                                                                                    |    0.000 |\n| 10187126 | watchdog        | 10.64.0.15:53464 | information_schema | Query   |     179 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10187127 | nagios          | localhost        | NULL               | Query   |     177 | init                             | show slave status                                                                                    |    0.000 |\n| 10187129 | nagios          | localhost        | NULL               | Query   |     176 | init                             | show slave status                                                                                    |    0.000 |\n| 10187130 | nagios          | localhost        | NULL               | Query   |     175 | init                             | show slave status                                                                                    |    0.000 |\n| 10187131 | nagios          | localhost        | NULL               | Query   |     174 | init                             | show slave status                                                                                    |    0.000 |\n| 10187132 | nagios          | localhost        | NULL               | Query   |     174 | init                             | show slave status                                                                                    |    0.000 |\n| 10187133 | nagios          | localhost        | NULL               | Query   |     174 | init                             | show slave status                                                                                    |    0.000 |\n| 10187135 | nagios          | localhost        | NULL               | Query   |     169 | init                             | show slave status                                                                                    |    0.000 |\n| 10187136 | nagios          | localhost        | NULL               | Query   |     169 | init                             | show slave status                                                                                    |    0.000 |\n| 10187137 | nagios          | localhost        | NULL               | Query   |     169 | init                             | show slave status                                                                                    |    0.000 |\n| 10187138 | nagios          | localhost        | NULL               | Query   |     169 | init                             | show slave status                                                                                    |    0.000 |\n| 10187139 | nagios          | localhost        | NULL               | Query   |     167 | init                             | show slave status                                                                                    |    0.000 |\n| 10187140 | nagios          | localhost        | NULL               | Query   |     167 | init                             | show slave status                                                                                    |    0.000 |\n| 10187144 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187145 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187146 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187147 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187148 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187149 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187150 | nagios          | localhost        | NULL               | Query   |     153 | init                             | show slave status                                                                                    |    0.000 |\n| 10187151 | nagios          | localhost        | NULL               | Query   |     152 | init                             | show slave status                                                                                    |    0.000 |\n| 10187152 | nagios          | localhost        | NULL               | Query   |     152 | init                             | show slave status                                                                                    |    0.000 |\n| 10187153 | nagios          | localhost        | NULL               | Query   |     152 | init                             | show slave status                                                                                    |    0.000 |\n| 10187155 | watchdog        | 10.64.0.15:54473 | information_schema | Query   |     149 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10187156 | nagios          | localhost        | NULL               | Query   |     149 | init                             | show slave status                                                                                    |    0.000 |\n| 10187157 | nagios          | localhost        | NULL               | Query   |     149 | init                             | show slave status                                                                                    |    0.000 |\n| 10187158 | nagios          | localhost        | NULL               | Query   |     149 | init                             | show slave status                                                                                    |    0.000 |\n| 10187159 | nagios          | localhost        | NULL               | Query   |     147 | init                             | show slave status                                                                                    |    0.000 |\n| 10187160 | nagios          | localhost        | NULL               | Query   |     147 | init                             | show slave status                                                                                    |    0.000 |\n| 10187161 | nagios          | localhost        | NULL               | Query   |     147 | init                             | show slave status                                                                                    |    0.000 |\n| 10187163 | nagios          | localhost        | NULL               | Query   |     145 | init                             | show slave status                                                                                    |    0.000 |\n| 10187164 | nagios          | localhost        | NULL               | Query   |     144 | init                             | show slave status                                                                                    |    0.000 |\n| 10187165 | nagios          | localhost        | NULL               | Query   |     144 | init                             | show slave status                                                                                    |    0.000 |\n| 10187167 | nagios          | localhost        | NULL               | Query   |     141 | init                             | show slave status                                                                                    |    0.000 |\n| 10187168 | nagios          | localhost        | NULL               | Query   |     141 | init                             | show slave status                                                                                    |    0.000 |\n| 10187169 | prometheus      | localhost        | NULL               | Query   |     139 | Filling schema table             | SHOW GLOBAL STATUS                                                                                   |    0.000 |\n| 10187170 | nagios          | localhost        | NULL               | Query   |     138 | init                             | show slave status                                                                                    |    0.000 |\n| 10187171 | nagios          | localhost        | NULL               | Query   |     138 | init                             | show slave status                                                                                    |    0.000 |\n| 10187173 | nagios          | localhost        | NULL               | Query   |     136 | init                             | show slave status                                                                                    |    0.000 |\n| 10187174 | nagios          | localhost        | NULL               | Query   |     136 | init                             | show slave status                                                                                    |    0.000 |\n| 10187175 | nagios          | localhost        | NULL               | Query   |     135 | init                             | show slave status                                                                                    |    0.000 |\n| 10187176 | nagios          | localhost        | NULL               | Query   |     135 | init                             | show slave status                                                                                    |    0.000 |\n| 10187177 | nagios          | localhost        | NULL               | Query   |     135 | init                             | show slave status                                                                                    |    0.000 |\n| 10187178 | nagios          | localhost        | NULL               | Query   |     135 | init                             | show slave status                                                                                    |    0.000 |\n| 10187179 | nagios          | localhost        | NULL               | Query   |     134 | init                             | show slave status                                                                                    |    0.000 |\n| 10187180 | nagios          | localhost        | NULL               | Query   |     132 | init                             | show slave status                                                                                    |    0.000 |\n| 10187182 | nagios          | localhost        | NULL               | Query   |     130 | init                             | show slave status                                                                                    |    0.000 |\n| 10187183 | prometheus      | localhost        | NULL               | Query   |     129 | Filling schema table             | SHOW GLOBAL STATUS                                                                                   |    0.000 |\n| 10187184 | nagios          | localhost        | NULL               | Query   |     129 | init                             | show slave status                                                                                    |    0.000 |\n| 10187185 | nagios          | localhost        | NULL               | Query   |     128 | init                             | show slave status                                                                                    |    0.000 |\n| 10187186 | nagios          | localhost        | NULL               | Query   |     128 | init                             | show slave status                                                                                    |    0.000 |\n| 10187187 | nagios          | localhost        | NULL               | Query   |     128 | init                             | show slave status                                                                                    |    0.000 |\n| 10187188 | nagios          | localhost        | NULL               | Query   |     128 | init                             | show slave status                                                                                    |    0.000 |\n| 10187189 | nagios          | localhost        | NULL               | Query   |     127 | init                             | show slave status                                                                                    |    0.000 |\n| 10187190 | nagios          | localhost        | NULL               | Query   |     127 | init                             | show slave status                                                                                    |    0.000 |\n| 10187192 | watchdog        | 10.64.0.15:55239 | information_schema | Query   |     100 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_VARIABLES`                                     |    0.000 |\n| 10187193 | nagios          | localhost        | NULL               | Query   |     125 | init                             | show slave status                                                                                    |    0.000 |\n| 10187194 | nagios          | localhost        | NULL               | Query   |     124 | init                             | show slave status                                                                                    |    0.000 |\n| 10187195 | nagios          | localhost        | NULL               | Query   |     123 | init                             | show slave status                                                                                    |    0.000 |\n| 10187196 | nagios          | localhost        | NULL               | Query   |     123 | init                             | show slave status                                                                                    |    0.000 |\n| 10187197 | nagios          | localhost        | NULL               | Query   |     122 | init                             | show slave status                                                                                    |    0.000 |\n| 10187198 | nagios          | localhost        | NULL               | Query   |     122 | init                             | show slave status                                                                                    |    0.000 |\n| 10187199 | nagios          | localhost        | NULL               | Query   |     122 | init                             | show slave status                                                                                    |    0.000 |\n| 10187200 | nagios          | localhost        | NULL               | Query   |     122 | init                             | show slave status                                                                                    |    0.000 |\n| 10187202 | nagios          | localhost        | NULL               | Query   |     119 | init                             | show slave status                                                                                    |    0.000 |\n| 10187203 | watchdog        | 10.64.0.15:55486 | information_schema | Query   |     119 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10187205 | nagios          | localhost        | NULL               | Query   |     116 | init                             | show slave status                                                                                    |    0.000 |\n| 10187206 | nagios          | localhost        | NULL               | Query   |     116 | init                             | show slave status                                                                                    |    0.000 |\n| 10187207 | nagios          | localhost        | NULL               | Query   |     115 | init                             | show slave status                                                                                    |    0.000 |\n| 10187208 | nagios          | localhost        | NULL               | Query   |     114 | init                             | show slave status                                                                                    |    0.000 |\n| 10187209 | nagios          | localhost        | NULL               | Query   |     114 | init                             | show slave status                                                                                    |    0.000 |\n| 10187210 | nagios          | localhost        | NULL               | Query   |     114 | init                             | show slave status                                                                                    |    0.000 |\n| 10187213 | nagios          | localhost        | NULL               | Query   |     109 | init                             | show slave status                                                                                    |    0.000 |\n| 10187214 | nagios          | localhost        | NULL               | Query   |     109 | init                             | show slave status                                                                                    |    0.000 |\n| 10187215 | nagios          | localhost        | NULL               | Query   |     109 | init                             | show slave status                                                                                    |    0.000 |\n| 10187216 | nagios          | localhost        | NULL               | Query   |     109 | init                             | show slave status                                                                                    |    0.000 |\n| 10187217 | nagios          | localhost        | NULL               | Query   |     107 | init                             | show slave status                                                                                    |    0.000 |\n| 10187218 | nagios          | localhost        | NULL               | Query   |     107 | init                             | show slave status                                                                                    |    0.000 |\n| 10187222 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187223 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187224 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187225 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187226 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187227 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187228 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187229 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187230 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187231 | nagios          | localhost        | NULL               | Query   |      92 | init                             | show slave status                                                                                    |    0.000 |\n| 10187233 | watchdog        | 10.64.0.15:56499 | information_schema | Query   |      89 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10187234 | nagios          | localhost        | NULL               | Query   |      88 | init                             | show slave status                                                                                    |    0.000 |\n| 10187235 | nagios          | localhost        | NULL               | Query   |      88 | init                             | show slave status                                                                                    |    0.000 |\n| 10187236 | nagios          | localhost        | NULL               | Query   |      88 | init                             | show slave status                                                                                    |    0.000 |\n| 10187237 | nagios          | localhost        | NULL               | Query   |      87 | init                             | show slave status                                                                                    |    0.000 |\n| 10187238 | nagios          | localhost        | NULL               | Query   |      87 | init                             | show slave status                                                                                    |    0.000 |\n| 10187239 | nagios          | localhost        | NULL               | Query   |      87 | init                             | show slave status                                                                                    |    0.000 |\n| 10187241 | nagios          | localhost        | NULL               | Query   |      85 | init                             | show slave status                                                                                    |    0.000 |\n| 10187242 | nagios          | localhost        | NULL               | Query   |      83 | init                             | show slave status                                                                                    |    0.000 |\n| 10187243 | nagios          | localhost        | NULL               | Query   |      83 | init                             | show slave status                                                                                    |    0.000 |\n| 10187245 | nagios          | localhost        | NULL               | Query   |      81 | init                             | show slave status                                                                                    |    0.000 |\n| 10187246 | nagios          | localhost        | NULL               | Query   |      81 | init                             | show slave status                                                                                    |    0.000 |\n| 10187247 | prometheus      | localhost        | NULL               | Query   |      78 | Filling schema table             | SHOW GLOBAL STATUS                                                                                   |    0.000 |\n| 10187248 | nagios          | localhost        | NULL               | Query   |      77 | init                             | show slave status                                                                                    |    0.000 |\n| 10187249 | nagios          | localhost        | NULL               | Query   |      77 | init                             | show slave status                                                                                    |    0.000 |\n| 10187251 | nagios          | localhost        | NULL               | Query   |      76 | init                             | show slave status                                                                                    |    0.000 |\n| 10187252 | nagios          | localhost        | NULL               | Query   |      76 | init                             | show slave status                                                                                    |    0.000 |\n| 10187253 | nagios          | localhost        | NULL               | Query   |      75 | init                             | show slave status                                                                                    |    0.000 |\n| 10187254 | nagios          | localhost        | NULL               | Query   |      75 | init                             | show slave status                                                                                    |    0.000 |\n| 10187255 | nagios          | localhost        | NULL               | Query   |      75 | init                             | show slave status                                                                                    |    0.000 |\n| 10187256 | nagios          | localhost        | NULL               | Query   |      75 | init                             | show slave status                                                                                    |    0.000 |\n| 10187257 | nagios          | localhost        | NULL               | Query   |      75 | init                             | show slave status                                                                                    |    0.000 |\n| 10187258 | nagios          | localhost        | NULL               | Query   |      72 | init                             | show slave status                                                                                    |    0.000 |\n| 10187260 | nagios          | localhost        | NULL               | Query   |      70 | init                             | show slave status                                                                                    |    0.000 |\n| 10187262 | nagios          | localhost        | NULL               | Query   |      69 | init                             | show slave status                                                                                    |    0.000 |\n| 10187263 | nagios          | localhost        | NULL               | Query   |      68 | init                             | show slave status                                                                                    |    0.000 |\n| 10187264 | nagios          | localhost        | NULL               | Query   |      68 | init                             | show slave status                                                                                    |    0.000 |\n| 10187265 | nagios          | localhost        | NULL               | Query   |      68 | init                             | show slave status                                                                                    |    0.000 |\n| 10187266 | nagios          | localhost        | NULL               | Query   |      68 | init                             | show slave status                                                                                    |    0.000 |\n| 10187267 | nagios          | localhost        | NULL               | Query   |      67 | init                             | show slave status                                                                                    |    0.000 |\n| 10187268 | nagios          | localhost        | NULL               | Query   |      67 | init                             | show slave status                                                                                    |    0.000 |\n| 10187270 | watchdog        | 10.64.0.15:57272 | information_schema | Query   |      40 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_VARIABLES`                                     |    0.000 |\n| 10187271 | nagios          | localhost        | NULL               | Query   |      65 | init                             | show slave status                                                                                    |    0.000 |\n| 10187272 | nagios          | localhost        | NULL               | Query   |      64 | init                             | show slave status                                                                                    |    0.000 |\n| 10187273 | nagios          | localhost        | NULL               | Query   |      64 | init                             | show slave status                                                                                    |    0.000 |\n| 10187274 | nagios          | localhost        | NULL               | Query   |      63 | init                             | show slave status                                                                                    |    0.000 |\n| 10187276 | nagios          | localhost        | NULL               | Query   |      61 | init                             | show slave status                                                                                    |    0.000 |\n| 10187277 | nagios          | localhost        | NULL               | Query   |      61 | init                             | show slave status                                                                                    |    0.000 |\n| 10187278 | nagios          | localhost        | NULL               | Query   |      61 | init                             | show slave status                                                                                    |    0.000 |\n| 10187279 | nagios          | localhost        | NULL               | Query   |      61 | init                             | show slave status                                                                                    |    0.000 |\n| 10187280 | nagios          | localhost        | NULL               | Query   |      59 | init                             | show slave status                                                                                    |    0.000 |\n| 10187281 | watchdog        | 10.64.0.15:57518 | information_schema | Query   |      59 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10187282 | root            | localhost        | NULL               | Killed  |      53 | init                             | show all slaves status                                                                               |    0.000 |\n| 10187284 | nagios          | localhost        | NULL               | Query   |      56 | init                             | show slave status                                                                                    |    0.000 |\n| 10187285 | nagios          | localhost        | NULL               | Query   |      55 | init                             | show slave status                                                                                    |    0.000 |\n| 10187286 | nagios          | localhost        | NULL               | Query   |      55 | init                             | show slave status                                                                                    |    0.000 |\n| 10187287 | nagios          | localhost        | NULL               | Query   |      53 | init                             | show slave status                                                                                    |    0.000 |\n| 10187288 | nagios          | localhost        | NULL               | Query   |      53 | init                             | show slave status                                                                                    |    0.000 |\n| 10187289 | nagios          | localhost        | NULL               | Query   |      53 | init                             | show slave status                                                                                    |    0.000 |\n| 10187291 | nagios          | localhost        | NULL               | Query   |      50 | init                             | show slave status                                                                                    |    0.000 |\n| 10187292 | nagios          | localhost        | NULL               | Query   |      50 | init                             | show slave status                                                                                    |    0.000 |\n| 10187293 | nagios          | localhost        | NULL               | Query   |      50 | init                             | show slave status                                                                                    |    0.000 |\n| 10187294 | nagios          | localhost        | NULL               | Query   |      50 | init                             | show slave status                                                                                    |    0.000 |\n| 10187295 | nagios          | localhost        | NULL               | Query   |      48 | init                             | show slave status                                                                                    |    0.000 |\n| 10187296 | nagios          | localhost        | NULL               | Query   |      47 | init                             | show slave status                                                                                    |    0.000 |\n| 10187302 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187303 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187304 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187305 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187306 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187307 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187308 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187309 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187310 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187311 | nagios          | localhost        | NULL               | Query   |      32 | init                             | show slave status                                                                                    |    0.000 |\n| 10187313 | watchdog        | 10.64.0.15:58527 | information_schema | Query   |      29 | Filling schema table             | SELECT `VARIABLE_NAME`, `VARIABLE_VALUE` FROM `GLOBAL_STATUS`                                        |    0.000 |\n| 10187314 | nagios          | localhost        | NULL               | Query   |      28 | init                             | show slave status                                                                                    |    0.000 |\n| 10187315 | nagios          | localhost        | NULL               | Query   |      28 | init                             | show slave status                                                                                    |    0.000 |\n| 10187316 | nagios          | localhost        | NULL               | Query   |      28 | init                             | show slave status                                                                                    |    0.000 |\n| 10187318 | nagios          | localhost        | NULL               | Query   |      26 | init                             | show slave status                                                                                    |    0.000 |\n| 10187319 | nagios          | localhost        | NULL               | Query   |      26 | init                             | show slave status                                                                                    |    0.000 |\n| 10187320 | nagios          | localhost        | NULL               | Query   |      26 | init                             | show slave status                                                                                    |    0.000 |\n| 10187321 | nagios          | localhost        | NULL               | Query   |      25 | init                             | show slave status                                                                                    |    0.000 |\n| 10187322 | nagios          | localhost        | NULL               | Query   |      24 | init                             | show slave status                                                                                    |    0.000 |\n| 10187323 | nagios          | localhost        | NULL               | Query   |      24 | init                             | show slave status                                                                                    |    0.000 |\n| 10187325 | nagios          | localhost        | NULL               | Query   |      21 | init                             | show slave status                                                                                    |    0.000 |\n| 10187326 | nagios          | localhost        | NULL               | Query   |      21 | init                             | show slave status                                                                                    |    0.000 |\n| 10187328 | nagios          | localhost        | NULL               | Query   |      17 | init                             | show slave status                                                                                    |    0.000 |\n| 10187329 | nagios          | localhost        | NULL               | Query   |      17 | init                             | show slave status                                                                                    |    0.000 |\n| 10187331 | nagios          | localhost        | NULL               | Query   |      16 | init                             | show slave status                                                                                    |    0.000 |\n| 10187332 | nagios          | localhost        | NULL               | Query   |      16 | init                             | show slave status                                                                                    |    0.000 |\n| 10187333 | nagios          | localhost        | NULL               | Query   |      15 | init                             | show slave status                                                                                    |    0.000 |\n| 10187334 | nagios          | localhost        | NULL               | Query   |      15 | init                             | show slave status                                                                                    |    0.000 |\n| 10187335 | nagios          | localhost        | NULL               | Query   |      15 | init                             | show slave status                                                                                    |    0.000 |\n| 10187336 | nagios          | localhost        | NULL               | Query   |      15 | init                             | show slave status                                                                                    |    0.000 |\n| 10187337 | nagios          | localhost        | NULL               | Query   |      15 | init                             | show slave status                                                                                    |    0.000 |\n| 10187339 | nagios          | localhost        | NULL               | Query   |      11 | init                             | show slave status                                                                                    |    0.000 |\n| 10187341 | nagios          | localhost        | NULL               | Query   |       9 | init                             | show slave status                                                                                    |    0.000 |\n| 10187342 | nagios          | localhost        | NULL               | Query   |       8 | init                             | show slave status                                                                                    |    0.000 |\n| 10187344 | nagios          | localhost        | NULL               | Query   |       8 | init                             | show slave status                                                                                    |    0.000 |\n| 10187345 | nagios          | localhost        | NULL               | Query   |       8 | init                             | show slave status                                                                                    |    0.000 |\n| 10187346 | nagios          | localhost        | NULL               | Query   |       8 | init                             | show slave status                                                                                    |    0.000 |\n| 10187347 | nagios          | localhost        | NULL               | Query   |       8 | init                             | show slave status                                                                                    |    0.000 |\n| 10187348 | root            | localhost        | NULL               | Sleep   |       3 |                                  | NULL                                                                                                 |    0.000 |\n| 10187349 | nagios          | localhost        | NULL               | Query   |       7 | init                             | show slave status                                                                                    |    0.000 |\n| 10187350 | nagios          | localhost        | NULL               | Query   |       7 | init                             | show slave status                                                                                    |    0.000 |\n| 10187352 | watchdog        | 10.64.0.15:59297 | information_schema | Sleep   |       6 |                                  | NULL                                                                                                 |    0.000 |\n| 10187353 | nagios          | localhost        | NULL               | Query   |       5 | init                             | show slave status                                                                                    |    0.000 |\n| 10187354 | root            | localhost        | NULL               | Query   |       0 | init                             | show processlist                                                                                     |    0.000 |\n| 10187355 | nagios          | localhost        | NULL               | Query   |       4 | init                             | show slave status                                                                                    |    0.000 |\n+----------+-----------------+------------------+--------------------+---------+---------+----------------------------------+------------------------------------------------------------------------------------------------------+----------+\n262 rows in set (0.00 sec)\nroot@DBSTORE[(none)]> kill 10187049;\nQuery OK, 0 rows affected (0.00 sec)\n\nroot@DBSTORE[(none)]> show processlist;\n+----------+-----------------+------------------+--------------------+---------+---------+----------------------------------+------------------------------------------------------------------------------------------------------+----------+\n| Id       | User            | Host             | db                 | Command | Time    | State                            | Info                                                                                                 | Progress |\n+----------+-----------------+------------------+--------------------+---------+---------+----------------------------------+------------------------------------------------------------------------------------------------------+----------+\n|  7927022 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927024 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927026 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927028 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927030 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927032 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927034 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927036 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927038 | system user     |                  | NULL               | Connect | 1354961 | Waiting for master to send event | NULL                                                                                                 |    0.000 |\n|  7927050 | event_scheduler | localhost        | NULL               | Daemon  |       0 | Waiting for next activation      | NULL                                                                                                 |    0.000 |\n| 10107012 | watchdog        | 10.64.0.15:48258 | mysql              | Sleep   |       5 |                                  | NULL                                                                                                 |    0.000 |\n| 10142049 | watchdog        | 10.64.0.15:34909 | information_schema | Sleep   |      85 |                                  | NULL                                                                                                 |    0.000 |\n| 10187348 | root            | localhost        | NULL               | Sleep   |       2 |                                  | NULL                                                                                                 |    0.000 |\n| 10187352 | watchdog        | 10.64.0.15:59297 | information_schema | Sleep   |      75 |                                  | NULL                                                                                                 |    0.000 |\n| 10187354 | root            | localhost        | NULL               | Query   |       0 | init                             | show processlist                                                                                     |    0.000 |\n| 10187443 | system user     |                  | NULL               | Connect |   86703 | closing tables                   | NULL                                                                                                 |    0.000 |\n| 10187444 | system user     |                  | cswiki             | Connect |   86899 | update                           | INSERT \/* LinksUpdate::incrTableUpdate  *\/ IGNORE INTO `templatelinks` (tl_from,tl_from_namespace,tl |    0.000 |\n| 10187445 | system user     |                  | kawiki             | Connect |   86676 | update                           | INSERT \/* Revision::insertOn  *\/  INTO `revision` (rev_id,rev_page,rev_text_id,rev_comment,rev_minor |    0.000 |\n| 10187446 | system user     |                  | NULL               | Connect |   86861 | closing tables                   | NULL                                                                                                 |    0.000 |\n| 10187447 | system user     |                  | dewiki             | Connect |   86810 | updating                         | UPDATE \/* HTMLCacheUpdateJob::invalidateTitles  *\/  `page` SET page_touched = 'sanitized' WHERE |    0.000 |\n| 10187448 | system user     |                  | ruwiki             | Connect |   86697 | updating                         | UPDATE \/* MediaWiki\\Auth\\TemporaryPasswordPrimaryAuthenticationProvider::providerChangeAuthenticatio |    0.000 |\n| 10187449 | system user     |                  | NULL               | Connect |   86787 | closing tables                   | NULL                                                                                                 |    0.000 |\n+----------+-----------------+------------------+--------------------+---------+---------+----------------------------------+------------------------------------------------------------------------------------------------------+----------+\n22 rows in set (0.00 sec)\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI have skipped that query, which is simply related to the pt-table-checksum ran from yesterday and restarted replication. I executed the checksum a few times, so we will see if it leaves the server in the same status for the next iterations. I will closely monitor it today.\u003c\/p\u003e\n\n\u003cp\u003eI wonder why it has not happened on dbstore1002 or dbstore2001 though.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_357\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_353\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_354\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_355\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3032477\" id=\"3032477\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3032477\" data-sigil=\"has-tooltip\" data-meta=\"0_351\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 16 2017, 9:42 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-16 09:42:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_349\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eGiven this on dbstore1001: \u003ctt class=\"remarkup-monospaced\"\u003e0 1 * * 3 \/usr\/local\/bin\/dumps-misc.sh >\/srv\/dumps-misc.log 2>&1\u003c\/tt\u003e means that it might have collided with a backup being taken at the same time as the replace was running for \u003ctt class=\"remarkup-monospaced\"\u003e77588\u003c\/tt\u003e seconds which is...21 hours.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_370\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_366\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_367\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_368\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_369\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3035718\" id=\"3035718\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3035718\" data-sigil=\"has-tooltip\" data-meta=\"0_364\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 6:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 06:48:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_362\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_363\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe dbstore2001 issue never happened again. It only happened once and I tested pt-table-checksum several times during the day so, if it was something in particular it should've happened at least 7-8 times during yesterday. So those are good news!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_379\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3035880\" id=\"3035880\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_378\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003egerritbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3035880\" data-sigil=\"has-tooltip\" data-meta=\"0_377\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 9:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 09:56:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_375\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_376\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 338326 had a related patch set uploaded (by Marostegui):\u003cbr \/\u003e\ngenerate_dsns_table.sh: Add new script for dsns.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/338326\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/338326\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_382\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3035881\" id=\"3035881\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3035881\" data-sigil=\"has-tooltip\" data-meta=\"0_380\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 9:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 09:56:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_395\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_391\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_393\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_394\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3035959\" id=\"3035959\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3035959\" data-sigil=\"has-tooltip\" data-meta=\"0_389\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 10:55 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 10:55:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_387\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have populated all the dsns tables, excluding db1069, db1095 and dbstore servers. Basically whatever is on the .hosts files but those hosts.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# for i in `mysql --skip-ssl dsns -e "show tables;" -BN`; do echo $i; mysql --skip-ssl dsns -e "select count(*) from $i;";done\ndsns_m3\n+----------+\n| count(*) |\n+----------+\n|        2 |\n+----------+\ndsns_s1\n+----------+\n| count(*) |\n+----------+\n|       19 |\n+----------+\ndsns_s2\n+----------+\n| count(*) |\n+----------+\n|       18 |\n+----------+\ndsns_s3\n+----------+\n| count(*) |\n+----------+\n|       11 |\n+----------+\ndsns_s4\n+----------+\n| count(*) |\n+----------+\n|       14 |\n+----------+\ndsns_s5\n+----------+\n| count(*) |\n+----------+\n|       13 |\n+----------+\ndsns_s6\n+----------+\n| count(*) |\n+----------+\n|       14 |\n+----------+\ndsns_s7\n+----------+\n| count(*) |\n+----------+\n|       14 |\n+----------+\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI have done so with a small shell script called generated_dsns_table.sh which will be pushed to the dbtools directory on the operations\/software repo: \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/#\/c\/338326\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/#\/c\/338326\/\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_404\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3035975\" id=\"3035975\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_403\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3035975\" data-sigil=\"has-tooltip\" data-meta=\"0_402\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 10:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 10:59:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_400\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 338326 merged by Jcrespo:\u003cbr \/\u003e\ngenerate_dsns_table.sh: Add new script for dsns.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/338326\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/338326\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_415\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_413\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_414\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3036450\" id=\"3036450\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_412\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3036450\" data-sigil=\"has-tooltip\" data-meta=\"0_411\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 1:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 13:49:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_409\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNot needed now, probably x1 will be needed. Also we should start thinking about merging the hosts lists into a dynamic table + maintaining that update, as when we decom. older hosts, old masters will shift to be slaves (and we will forget to change all this tables :-\/). This is more of the inventory + automatic checking tasks, not a concern right now.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_429\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_425\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_427\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3036523\" id=\"3036523\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#3036523\" data-sigil=\"has-tooltip\" data-meta=\"0_423\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 2:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 14:42:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_421\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_422\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI am thinking about running this first iteration over s2 and a concrete database next week at some point: nlwiki. (s2)\u003cbr \/\u003e\nWhy nlwiki? because it is around 100G and the biggest table it has is revision with 23G, which can be a good test.\u003c\/p\u003e\n\n\u003cp\u003eFrom the tables it has the following tables would need to be ignored (as they do not have a PK, and the list is the same on all the hosts on s2):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ecategorylinks\nchange_tag\nclick_tracking\nclick_tracking_user_properties\ncur\nedit_page_tracking\nep_users_per_course\nhidden\nimagelinks\ninterwiki\niwlinks\nl10n_cache\nlanglinks\nlinkscc\nlog_search\nlogging_pre_1_10\nmath\nmodule_deps\nmsg_resource\nmsg_resource_links\nobjectcache\noldimage\npagelinks\nprefstats\nprefswitch_survey\nquerycache\nquerycache_info\nquerycachetwo\nsecurepoll_lists\nsecurepoll_msgs\nsecurepoll_properties\nsite_identifiers\nsite_stats\ntag_summary\ntemplatelinks\ntext\ntranscache\nuser_former_groups\nuser_newtalk\nuser_properties\nwatchlist\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eExcluding that list, we'd still have revision (23G) and logging (17G) as good tests.\u003cbr \/\u003e\nIf we generated delay, the following wikis would be affected:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ebgwiki\ncswiki\nenwikiquote\neowiki\nfiwiki\nidwiki\nitwiki\nl10nwiki\nnlwiki\nnowiki\nplwiki\nptwiki\nsvwiki\nthwiki\ntrwiki\nzhwiki\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI will probably run another test on m3 with a chunk-size of 100 instead of 200 and see if we get even less delay that the one we got (reminder: peak of 12-13 seconds for around 20-30 seconds)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_440\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_438\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_439\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3036527\" id=\"3036527\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003ejcrespo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3036527\" data-sigil=\"has-tooltip\" data-meta=\"0_436\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 17 2017, 2:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-17 14:49:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_434\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe have semisync on core, I would expect much less impact, lag-wise.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_454\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_450\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_452\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3040625\" id=\"3040625\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle handle-availability-partial phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Marostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#3040625\" data-sigil=\"has-tooltip\" data-meta=\"0_448\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 20 2017, 1:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-20 13:48:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_446\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_447\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe idea would be to run the pt-table-checksum with the following tables on the same batch for nlwiki:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e-rw-rw---- 1 mysql mysql 112K Feb 20  2014 ep_instructors.ibd\n-rw-rw---- 1 mysql mysql 112K Feb 11 17:38 babel.ibd\n-rw-rw---- 1 mysql mysql 128K Mar 16  2014 ep_oas.ibd\n-rw-rw---- 1 mysql mysql 128K Mar 16  2014 ep_cas.ibd\n-rw-rw---- 1 mysql mysql 144K Oct 21 14:27 ores_model.ibd\n-rw-rw---- 1 mysql mysql 176K Feb 18 18:32 image.ibd\n-rw-rw---- 1 mysql mysql 176K Mar 31  2015 ep_articles.ibd\n-rw-rw---- 1 mysql mysql 208K May 24  2015 ep_students.ibd\n-rw-rw---- 1 mysql mysql 240K Mar 27  2015 ep_orgs.ibd\n-rw-rw---- 1 mysql mysql 272K Mar 27  2015 ep_revisions.ibd\n-rw-rw---- 1 mysql mysql 320K Mar 27  2015 ep_courses.ibd\n-rw-rw---- 1 mysql mysql 784K Dec  1 03:26 sites.ibd\n-rw-rw---- 1 mysql mysql 1.0M Nov 13 11:42 valid_tag.ibd\n-rw-rw---- 1 mysql mysql 1.0M Nov 28 13:58 uploadstash.ibd\n-rw-rw---- 1 mysql mysql 1.0M Jan  6 06:10 updatelog.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 transcode.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 trackbacks.ibd\n-rw-rw---- 1 mysql mysql 1.0M Feb 19 10:35 protected_titles.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 pr_index.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 povwatch_subscribers.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 povwatch_log.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 job.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 global_block_whitelist.ibd\n-rw-rw---- 1 mysql mysql 1.0M Sep 25  2013 filejournal.ibd\n-rw-rw---- 1 mysql mysql 1.0M Feb 20 13:14 betafeatures_user_counts.ibd\n-rw-rw---- 1 mysql mysql 1.0M Feb 20 12:22 abuse_filter.ibd\n-rw-rw---- 1 mysql mysql 1.0M Feb 19 16:07 abuse_filter_action.ibd\n-rw-rw---- 1 mysql mysql 8.0M Feb 17 19:47 user_groups.ibd\n-rw-rw---- 1 mysql mysql 9.0M Sep 24  2013 pif_edits.ibd\n-rw-rw---- 1 mysql mysql 9.0M Nov 27 02:58 moodbar_feedback_response.ibd\n-rw-rw---- 1 mysql mysql 9.0M Feb 19 16:07 abuse_filter_history.ibd\n-rw-rw---- 1 mysql mysql  10M Feb 19 15:55 page_restrictions.ibd\n-rw-rw---- 1 mysql mysql  11M May 24  2015 ep_events.ibd\n-rw-rw---- 1 mysql mysql  12M Dec 13 03:20 moodbar_feedback.ibd\n-rw-rw---- 1 mysql mysql  12M Feb 19 19:56 cu_log.ibd\n-rw-rw---- 1 mysql mysql  16M Sep 24  2013 bv2009_edits.ibd\n-rw-rw---- 1 mysql mysql  20M Sep 24  2013 bv2011_edits.ibd\n-rw-rw---- 1 mysql mysql  24M Sep 24  2013 blob_orphans.ibd\n-rw-rw---- 1 mysql mysql  29M May  4  2015 bv2015_edits.ibd\n-rw-rw---- 1 mysql mysql  30M Feb 20 13:44 category.ibd\n-rw-rw---- 1 mysql mysql  31M Feb 20 13:15 ipblocks.ibd\n-rw-rw---- 1 mysql mysql  36M Feb 20 13:43 geo_tags.ibd\n-rw-rw---- 1 mysql mysql  76M Feb 20 13:42 spoofuser.ibd\n-rw-rw---- 1 mysql mysql  80M Feb 20 13:09 ores_classification.ibd\n-rw-rw---- 1 mysql mysql  88M Feb 18 18:32 filearchive.ibd\n-rw-rw---- 1 mysql mysql 108M Feb 20 13:29 redirect.ibd\n-rw-rw---- 1 mysql mysql 304M Feb 20 13:44 user.ibd\n-rw-rw---- 1 mysql mysql 404M Feb 20 13:40 abuse_filter_log.ibd\n-rw-rw---- 1 mysql mysql 496M Oct 31  2014 titlekey.ibd\n-rw-rw---- 1 mysql mysql 604M Mar 15  2016 updates.ibd\n-rw-rw---- 1 mysql mysql 672M Feb 20 13:34 page_props.ibd\n-rw-rw---- 1 mysql mysql 800M Feb 20 13:16 wbc_entity_usage.ibd\n-rw-rw---- 1 mysql mysql 1.1G Feb 20 13:45 page.ibd\n-rw-rw---- 1 mysql mysql 1.1G Sep 24  2013 blob_tracking.ibd\n-rw-rw---- 1 mysql mysql 1.3G Feb 20 13:39 archive.ibd\n-rw-rw---- 1 mysql mysql 1.7G Feb 20 13:45 recentchanges.ibd\n-rw-rw---- 1 mysql mysql 1.7G Feb 20 13:45 cu_changes.ibd\n-rw-rw---- 1 mysql mysql 5.0G Feb 20 13:44 externallinks.ibd\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd then run an individual "per table" check for the two biggest tables on different runs:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e-rw-rw---- 1 mysql mysql  17G Feb 20 13:44 logging.ibd\n-rw-rw---- 1 mysql mysql  23G Feb 20 13:45 revision.ibd\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe idea of doing it in different batches is to try to minimize possible lags and overload, at least on this initial tests till we are sure how to tweak the runs and the table sizes.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_463\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3045895\" id=\"3045895\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_462\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3045895\" data-sigil=\"has-tooltip\" data-meta=\"0_461\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 22 2017, 9:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-02-22 09:14:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_459\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_460\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AVplGdSOQMK9DA-FKiwE\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2017-02-22T09:14:36Z] <marostegui> Run pt-table-checksum on s2.nlwiki over some tables - \u003ca href=\"\/T154485\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT154485\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-tzam6pe467jitnjaltsc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-tzam6pe467jitnjaltsc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yv24a2dbcpysaunw7qtn"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-2nofvqs5pssdedl"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-xfxg55miklfe6ll"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-cbr3f72reikqsynhfgjv"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bk6sgdzdhctxupi3vn5b"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-54xjtvb6c5vxcsr2izkm"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-46ti3cthb2ssoitbjzqg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-kecacj3oii4po6a"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-7cj5ijfw7wwmhzn"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-okuganq3vlmcgc3"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-3qswkp2rlhfbmeg"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-axm5clnts35wxldvycvl"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-cbr3f72reikqsynhfgjv"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-wchcaamunjhj3gp"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-rwn5mznwutyhw5m"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-yemlt64exkncju3"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-2yazk5bsyv5dw6v"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-5vrkv43wvxm56bd"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-rmp2pwh54cii7sd"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-ryvstydllvsict6"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-lhzwewhgzmtssw5"},{"phid":"PHID-XACT-TASK-2atusia3ky6hht4"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-i6rwhyrbg5alfuu"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-ns4le7x4e7vewep"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-v2c6yml7xmrv66q"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-3a4uwu2ibbym45f"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-avtzczff42cvkls"},{"phid":"PHID-XACT-TASK-mn3kgpcvbkyivid"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-hl23dzaug4jfasw"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-itjlziytoi6wwtj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-5wbvovaadu4jyak"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-ndayo5tdhwzuneh"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"phid":"PHID-XACT-TASK-tdcvkkfoo4uqxbo"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-cnvra627t6zgyql"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-iw2lqnxlfcpcrpf"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-n3blbpvrrfxease","anchor":"2912830"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-p6mcqr7zqidqmus","anchor":"2912843"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-se5dvc6x2pgxkn4","anchor":"2912844"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2nofvqs5pssdedl\/","ref":"T154485#2916083"},[],{"anchor":"2916083"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_109\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_110\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2nofvqs5pssdedl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_111\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_112\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-2nofvqs5pssdedl","anchor":"2916083"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xfxg55miklfe6ll\/","ref":"T154485#2919046"},[],{"anchor":"2919046"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_122\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xfxg55miklfe6ll\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_124\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-xfxg55miklfe6ll","anchor":"2919046"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wecfe6xvv3oa6hs","anchor":"2928739"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ckf5mxr2rhyxkn4","anchor":"2934674"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4wpyb6snfq7iys7","anchor":"2992332"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kecacj3oii4po6a\/","ref":"T154485#3024248"},[],{"anchor":"3024248"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kecacj3oii4po6a\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_146\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-kecacj3oii4po6a","anchor":"3024248"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7cj5ijfw7wwmhzn\/","ref":"T154485#3024425"},[],{"anchor":"3024425"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_157\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7cj5ijfw7wwmhzn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_159\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_160\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-7cj5ijfw7wwmhzn","anchor":"3024425"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-okuganq3vlmcgc3\/","ref":"T154485#3024428"},[],{"anchor":"3024428"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_168\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-okuganq3vlmcgc3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_170\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-okuganq3vlmcgc3","anchor":"3024428"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3qswkp2rlhfbmeg\/","ref":"T154485#3025051"},[],{"anchor":"3025051"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_181\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3qswkp2rlhfbmeg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_183\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3qswkp2rlhfbmeg","anchor":"3025051"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-54ptivpgrp2v7ql","anchor":"3025449"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wchcaamunjhj3gp\/","ref":"T154485#3028719"},[],{"anchor":"3028719"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_198\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_199\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wchcaamunjhj3gp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_200\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-wchcaamunjhj3gp","anchor":"3028719"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rwn5mznwutyhw5m\/","ref":"T154485#3028736"},[],{"anchor":"3028736"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_211\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rwn5mznwutyhw5m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_213\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_214\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rwn5mznwutyhw5m","anchor":"3028736"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yemlt64exkncju3\/","ref":"T154485#3028741"},[],{"anchor":"3028741"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_222\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_223\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yemlt64exkncju3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_224\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-yemlt64exkncju3","anchor":"3028741"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2yazk5bsyv5dw6v\/","ref":"T154485#3028742"},[],{"anchor":"3028742"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_233\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2yazk5bsyv5dw6v\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_235\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-2yazk5bsyv5dw6v\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-2yazk5bsyv5dw6v","anchor":"3028742"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5vrkv43wvxm56bd\/","ref":"T154485#3028759"},[],{"anchor":"3028759"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_245\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5vrkv43wvxm56bd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_247\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-5vrkv43wvxm56bd","anchor":"3028759"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rmp2pwh54cii7sd\/","ref":"T154485#3028802"},[],{"anchor":"3028802"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_258\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_259\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rmp2pwh54cii7sd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_260\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_261\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rmp2pwh54cii7sd","anchor":"3028802"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ryvstydllvsict6\/","ref":"T154485#3028813"},[],{"anchor":"3028813"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_269\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_270\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ryvstydllvsict6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_271\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_272\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ryvstydllvsict6","anchor":"3028813"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lhzwewhgzmtssw5\/","ref":"T154485#3028822"},[],{"anchor":"3028822"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_280\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lhzwewhgzmtssw5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_282\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-lhzwewhgzmtssw5","anchor":"3028822"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2atusia3ky6hht4\/","ref":"T154485#3028895"},[],{"anchor":"3028895"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_293\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2atusia3ky6hht4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_295\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2atusia3ky6hht4","anchor":"3028895"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-i6rwhyrbg5alfuu\/","ref":"T154485#3028898"},[],{"anchor":"3028898"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_302\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-i6rwhyrbg5alfuu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_304\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-i6rwhyrbg5alfuu","anchor":"3028898"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-rdszslqw4cay2rf","anchor":"3028952"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ns4le7x4e7vewep\/","ref":"T154485#3030925"},[],{"anchor":"3030925"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_319\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ns4le7x4e7vewep\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_321\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_322\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ns4le7x4e7vewep","anchor":"3030925"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-v2c6yml7xmrv66q\/","ref":"T154485#3032453"},[],{"anchor":"3032453"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_332\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_333\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v2c6yml7xmrv66q\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_334\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_335\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-v2c6yml7xmrv66q","anchor":"3032453"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3a4uwu2ibbym45f\/","ref":"T154485#3032477"},[],{"anchor":"3032477"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_345\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3a4uwu2ibbym45f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_347\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_348\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3a4uwu2ibbym45f","anchor":"3032477"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-avtzczff42cvkls\/","ref":"T154485#3035718"},[],{"anchor":"3035718"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_358\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-avtzczff42cvkls\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_360\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_361\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-avtzczff42cvkls","anchor":"3035718"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mn3kgpcvbkyivid\/","ref":"T154485#3035880"},[],{"anchor":"3035880"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_371\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_372\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mn3kgpcvbkyivid\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_373\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mn3kgpcvbkyivid","anchor":"3035880"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-au7cwz434brdv6u","anchor":"3035881"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hl23dzaug4jfasw\/","ref":"T154485#3035959"},[],{"anchor":"3035959"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_383\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_384\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hl23dzaug4jfasw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_385\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-hl23dzaug4jfasw","anchor":"3035959"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-itjlziytoi6wwtj\/","ref":"T154485#3035975"},[],{"anchor":"3035975"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_396\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_397\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-itjlziytoi6wwtj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_398\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-itjlziytoi6wwtj","anchor":"3035975"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5wbvovaadu4jyak\/","ref":"T154485#3036450"},[],{"anchor":"3036450"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_405\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_406\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5wbvovaadu4jyak\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_407\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_408\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-5wbvovaadu4jyak","anchor":"3036450"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ndayo5tdhwzuneh\/","ref":"T154485#3036523"},[],{"anchor":"3036523"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_416\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_417\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ndayo5tdhwzuneh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_418\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-ndayo5tdhwzuneh\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_420\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ndayo5tdhwzuneh","anchor":"3036523"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tdcvkkfoo4uqxbo\/","ref":"T154485#3036527"},[],{"anchor":"3036527"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_430\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tdcvkkfoo4uqxbo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_432\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-tdcvkkfoo4uqxbo","anchor":"3036527"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cnvra627t6zgyql\/","ref":"T154485#3040625"},[],{"anchor":"3040625"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_441\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cnvra627t6zgyql\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_443\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-cnvra627t6zgyql\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_445\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-cnvra627t6zgyql","anchor":"3040625"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-iw2lqnxlfcpcrpf\/","ref":"T154485#3045895"},[],{"anchor":"3045895"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_455\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_456\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-iw2lqnxlfcpcrpf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_457\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-iw2lqnxlfcpcrpf","anchor":"3045895"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}