for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-jvhyszid3tb7h5dkgwcz\/?after=4397775&quoteTargetID=UQ0_1&quoteRef=T120242\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_219\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/zirzntnbnbimsxdels47\/PHID-FILE-tu3ola3spvpugy5kzs6b\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Mholloway\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4397775\" id=\"4397775\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Mholloway\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Mholloway\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4397775\" data-sigil=\"has-tooltip\" data-meta=\"0_217\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2018, 3:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-04 15:17:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_222\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/biyialftej5zydh2v4t7\/PHID-FILE-hadstttxv543p5lf44xo\/profile-me.jpeg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Milimetric\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4400363\" id=\"4400363\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_221\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Milimetric\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003eMilimetric\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/11\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eIncoming\u003c\/a\u003e to \u003ca href=\"\/project\/board\/11\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003eEvent Platform\u003c\/a\u003e on the \u003ca href=\"\/tag\/analytics\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eAnalytics\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4400363\" data-sigil=\"has-tooltip\" data-meta=\"0_220\"\u003e\u003cspan class=\"screen-only\"\u003eJul 5 2018, 4:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-05 16:36:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_226\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/sql6zxypsdgnvcij2wjr\/PHID-FILE-pgi2xyqugx6rvnlyptyo\/648d72-alphanumeric_lato-dark_P.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Pchelolo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4474702\" id=\"4474702\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_224\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Pchelolo\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Pchelolo\u003c\/a\u003e mentioned this in \u003ca href=\"\/T201068\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eT201068: Modern Event Platform: Stream Intake Service\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4474702\" data-sigil=\"has-tooltip\" data-meta=\"0_223\"\u003e\u003cspan class=\"screen-only\"\u003eAug 2 2018, 9:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-08-02 21:14:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Pchelolo\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Pchelolo\u003c\/a\u003e added a parent task: \u003ca href=\"\/T201068\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003eT201068: Modern Event Platform: Stream Intake Service\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_229\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/mcdy6bokf2bfd6lhrj5x\/PHID-FILE-hvay22mxxy4ab7u76frl\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Addshore\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4518712\" id=\"4518712\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Addshore\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003eAddshore\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4518712\" data-sigil=\"has-tooltip\" data-meta=\"0_227\"\u003e\u003cspan class=\"screen-only\"\u003eAug 21 2018, 12:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-08-21 12:52:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_232\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4837384\" id=\"4837384\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added a project: \u003ca href=\"\/project\/view\/3635\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003ePlatform Team Legacy (Later)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4837384\" data-sigil=\"has-tooltip\" data-meta=\"0_230\"\u003e\u003cspan class=\"screen-only\"\u003eDec 20 2018, 12:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-20 12:07:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_235\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6054962\" id=\"6054962\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eOttomata\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1474\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1474\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003eRadar\u003c\/a\u003e on the \u003ca href=\"\/tag\/event-platform\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003eEvent-Platform\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6054962\" data-sigil=\"has-tooltip\" data-meta=\"0_233\"\u003e\u003cspan class=\"screen-only\"\u003eApr 14 2020, 1:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-04-14 13:26:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_238\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6055221\" id=\"6055221\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-times phui-timeline-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eKrinkle\u003c\/a\u003e unsubscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6055221\" data-sigil=\"has-tooltip\" data-meta=\"0_236\"\u003e\u003cspan class=\"screen-only\"\u003eApr 14 2020, 2:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-04-14 14:50:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_241\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6471024\" id=\"6471024\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eOttomata\u003c\/a\u003e mentioned this in \u003ca href=\"\/T215001\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003eT215001: Revisions missing from mediawiki_revision_create\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6471024\" data-sigil=\"has-tooltip\" data-meta=\"0_239\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2020, 3:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-09-17 15:34:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_251\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6471045\" id=\"6471045\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_250\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6471045\" data-sigil=\"has-tooltip\" data-meta=\"0_249\"\u003e\u003cspan class=\"screen-only\"\u003eSep 17 2020, 3:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-09-17 15:40:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_247\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_248\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_72\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe might be able to achieve this with a Kafka client in MW that uses \u003ca href=\"https:\/\/www.confluent.io\/blog\/transactional-systems-with-apache-kafka\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eKafka transactions\u003c\/a\u003e.  But then, we'd need a good PHP Kafka client, and would likely have to fire up a new Kafka producer connection with every MW request, which is not very efficient.  We'd also have to some how tie the Kafka produce call with the MySQL DB write call into a transaction.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_257\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6669518\" id=\"6669518\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-wsywz4iudqyfe7z\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6669518\" data-sigil=\"has-tooltip\" data-meta=\"0_252\"\u003e\u003cspan class=\"screen-only\"\u003eDec 4 2020, 4:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-04 16:06:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_254\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-4qf4xgi2df35q24\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-wgcsbf6nlijksvy\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-ducsr42lgpb62jm\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_266\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6669528\" id=\"6669528\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6669528\" data-sigil=\"has-tooltip\" data-meta=\"0_264\"\u003e\u003cspan class=\"screen-only\"\u003eDec 4 2020, 4:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-04 16:10:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_262\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_263\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_78\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWe'd also have to some how tie the Kafka produce call with the MySQL DB write call into a transaction.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eTo do this I think we'd need some kind of two phase commit service for MediaWiki, which sounds really hard to me!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_270\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6669543\" id=\"6669543\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_268\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eOttomata\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/platform_engineering\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003ePlatform Engineering\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6669543\" data-sigil=\"has-tooltip\" data-meta=\"0_267\"\u003e\u003cspan class=\"screen-only\"\u003eDec 4 2020, 4:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-04 16:12:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003eOttomata\u003c\/a\u003e added subscribers: \u003ca href=\"\/p\/Pchelolo\/\" class=\"phui-handle handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Pchelolo\u003c\/a\u003e, \u003ca href=\"\/p\/dianamontalion\/\" class=\"phui-handle handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dianamontalion\u003c\/a\u003e, \u003ca href=\"\/p\/kchapman\/\" class=\"phui-handle handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e kchapman\u003c\/a\u003e and \u003ca href=\"\/subscriptions\/transaction\/add\/PHID-XACT-TASK-px6tysjb2czvrgb\/\" data-sigil=\"workflow\"\u003e3 others\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_279\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6669547\" id=\"6669547\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6669547\" data-sigil=\"has-tooltip\" data-meta=\"0_277\"\u003e\u003cspan class=\"screen-only\"\u003eDec 4 2020, 4:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-04 16:14:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_275\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/dianamontalion\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_1\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@dianamontalion\u003c\/span\u003e\u003c\/a\u003e \u003cbr \/\u003e\nI updated this task with what I hope is more descriptive of the problem and some possible solutions.  I really think solving this is key to the Architecture teams mission of supporting 'infinite use cases \u003cem\u003einsert rest of arch mission phraseology here\u003c\/em\u003e' :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_282\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6669548\" id=\"6669548\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003eOttomata\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/JAllemandou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003eJAllemandou\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6669548\" data-sigil=\"has-tooltip\" data-meta=\"0_280\"\u003e\u003cspan class=\"screen-only\"\u003eDec 4 2020, 4:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-04 16:14:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_286\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/fx4bdiqe6cjw224wk3x7\/PHID-FILE-fiw6mqnlgfuajewlfzs5\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/JAllemandou\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6672474\" id=\"6672474\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-trophy phui-timeline-icon\" data-meta=\"0_284\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view sprite-tokens tokens-coin-4 phui-timeline-token\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003ca href=\"\/p\/JAllemandou\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eJAllemandou\u003c\/a\u003e awarded a token.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6672474\" data-sigil=\"has-tooltip\" data-meta=\"0_283\"\u003e\u003cspan class=\"screen-only\"\u003eDec 7 2020, 8:41 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-07 08:41:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_290\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/biyialftej5zydh2v4t7\/PHID-FILE-hadstttxv543p5lf44xo\/profile-me.jpeg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Milimetric\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6673764\" id=\"6673764\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-trophy phui-timeline-icon\" data-meta=\"0_288\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view sprite-tokens tokens-coin-4 phui-timeline-token\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Milimetric\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_87\"\u003eMilimetric\u003c\/a\u003e awarded a token.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6673764\" data-sigil=\"has-tooltip\" data-meta=\"0_287\"\u003e\u003cspan class=\"screen-only\"\u003eDec 7 2020, 4:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-07 16:56:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_293\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/vwxgvshwb7nxsatweel6\/PHID-FILE-vvklnm2rqx6mm6hx3btq\/335862-alphanumeric_lato-white_C.png-0%2C0%2C0%2C0.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Clarakosi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6677516\" id=\"6677516\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_292\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Clarakosi\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Clarakosi\u003c\/a\u003e edited projects, added \u003ca href=\"\/tag\/wmf-architecture-team\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_89\"\u003eWMF-Architecture-Team\u003c\/a\u003e; removed \u003ca href=\"\/tag\/platform_engineering\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003ePlatform Engineering\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6677516\" data-sigil=\"has-tooltip\" data-meta=\"0_291\"\u003e\u003cspan class=\"screen-only\"\u003eDec 8 2020, 9:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-08 21:27:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_302\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6678059\" id=\"6678059\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6678059\" data-sigil=\"has-tooltip\" data-meta=\"0_300\"\u003e\u003cspan class=\"screen-only\"\u003eDec 8 2020, 11:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-08 23:28:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_298\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_299\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"phabricator-remarkup-mention-unknown\"\u003e@claroski\u003c\/span\u003e o\/ Curious, is Platform Engineering not a relevant tag?  Work on this would very likely affect things like ChangeProp and JobQueue.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_305\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6699804\" id=\"6699804\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_304\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-pkljqre3z75ypqc\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6699804\" data-sigil=\"has-tooltip\" data-meta=\"0_303\"\u003e\u003cspan class=\"screen-only\"\u003eDec 17 2020, 9:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-17 21:13:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_309\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6710348\" id=\"6710348\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-livgdrtet2zenva\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6710348\" data-sigil=\"has-tooltip\" data-meta=\"0_306\"\u003e\u003cspan class=\"screen-only\"\u003eDec 23 2020, 3:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-23 15:42:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_95\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-ceqjubyg45fr7p6\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_312\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6710691\" id=\"6710691\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-dl5lhwjog6fknfn\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6710691\" data-sigil=\"has-tooltip\" data-meta=\"0_310\"\u003e\u003cspan class=\"screen-only\"\u003eDec 23 2020, 6:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-23 18:16:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_315\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6722700\" id=\"6722700\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-ovx4grepqa4ugut\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6722700\" data-sigil=\"has-tooltip\" data-meta=\"0_313\"\u003e\u003cspan class=\"screen-only\"\u003eJan 5 2021, 4:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-05 16:12:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_324\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/biyialftej5zydh2v4t7\/PHID-FILE-hadstttxv543p5lf44xo\/profile-me.jpeg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Milimetric\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6741183\" id=\"6741183\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Milimetric\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003eMilimetric\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Clarakosi\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_99\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Clarakosi\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6741183\" data-sigil=\"has-tooltip\" data-meta=\"0_322\"\u003e\u003cspan class=\"screen-only\"\u003eJan 12 2021, 7:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-12 19:48:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_320\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_100\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Clarakosi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Clarakosi\u003c\/span\u003e\u003c\/a\u003e: I think \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e meant to ping you above, adding here.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_333\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/vwxgvshwb7nxsatweel6\/PHID-FILE-vvklnm2rqx6mm6hx3btq\/335862-alphanumeric_lato-white_C.png-0%2C0%2C0%2C0.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Clarakosi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6741415\" id=\"6741415\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Clarakosi\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Clarakosi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6741415\" data-sigil=\"has-tooltip\" data-meta=\"0_331\"\u003e\u003cspan class=\"screen-only\"\u003eJan 12 2021, 8:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-12 20:31:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_329\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_102\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6741183\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6741183\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Milimetric\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Milimetric\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Clarakosi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Clarakosi\u003c\/span\u003e\u003c\/a\u003e: I think \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e meant to ping you above, adding here.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThanks!\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6678059\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6678059\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003cspan class=\"phabricator-remarkup-mention-unknown\"\u003e@claroski\u003c\/span\u003e o\/ Curious, is Platform Engineering not a relevant tag?  Work on this would very likely affect things like ChangeProp and JobQueue.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI think I understood it as needing feedback from the Architecture team first but retagging with \u003ca href=\"\/tag\/platform_engineering_roadmap_decision_making\/\" class=\"phui-tag-view phui-tag-type-shade phui-tag-disabled phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-users\" data-meta=\"0_10\" aria-hidden=\"true\"\u003e\u003c\/span\u003ePlatform Engineering Roadmap Decision Making\u003c\/span\u003e\u003c\/a\u003e for review.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_342\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6741422\" id=\"6741422\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_103\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6741422\" data-sigil=\"has-tooltip\" data-meta=\"0_340\"\u003e\u003cspan class=\"screen-only\"\u003eJan 12 2021, 8:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-12 20:33:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_338\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_104\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAt the moment I don't think there are any actionables, but there might be in a couple of quarters.  Not sure if there is a decision to be made yet, but there will be eventually!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_351\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6763163\" id=\"6763163\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_105\"\u003eOttomata\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003eKrinkle\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6763163\" data-sigil=\"has-tooltip\" data-meta=\"0_349\"\u003e\u003cspan class=\"screen-only\"\u003eJan 20 2021, 7:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-20 19:23:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_347\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_348\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFYI, I had a chat with \u003ca href=\"\/p\/Krinkle\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Krinkle\u003c\/span\u003e\u003c\/a\u003e yesterday, and he informed be that for all MediaWIki browser client generated writes to MediaWiki MySQL are wrapped in a transaction anyway!  So, a Transactional Outbox solution would most likely be possible without any changes to MediaWiki core.  We should follow up on how this is different for API requests, maintenance, or JobQueue based writes, as we'd likely have to have those wrapped in a transaction too.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_354\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6765826\" id=\"6765826\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_353\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-wb55zvj6zpmg63g\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6765826\" data-sigil=\"has-tooltip\" data-meta=\"0_352\"\u003e\u003cspan class=\"screen-only\"\u003eJan 21 2021, 4:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-21 16:07:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_357\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6765832\" id=\"6765832\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003eOttomata\u003c\/a\u003e renamed this task from \u003cspan class=\"phui-timeline-value\"\u003eReliable (atomic) MediaWiki event production\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eReliable (atomic) MediaWiki event production \/ MediaWiki events as source of truth\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6765832\" data-sigil=\"has-tooltip\" data-meta=\"0_355\"\u003e\u003cspan class=\"screen-only\"\u003eJan 21 2021, 4:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-21 16:11:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_360\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/biyialftej5zydh2v4t7\/PHID-FILE-hadstttxv543p5lf44xo\/profile-me.jpeg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Milimetric\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6766371\" id=\"6766371\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Milimetric\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003eMilimetric\u003c\/a\u003e added a subtask: \u003ca href=\"\/T215001\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003eT215001: Revisions missing from mediawiki_revision_create\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6766371\" data-sigil=\"has-tooltip\" data-meta=\"0_358\"\u003e\u003cspan class=\"screen-only\"\u003eJan 21 2021, 6:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-21 18:21:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_363\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ye6hvtalj6vufghp7iln\/PHID-FILE-pqd4ihxwjtcyoweb6mge\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gmodena\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6785438\" id=\"6785438\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gmodena\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003egmodena\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6785438\" data-sigil=\"has-tooltip\" data-meta=\"0_361\"\u003e\u003cspan class=\"screen-only\"\u003eJan 28 2021, 9:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-28 21:32:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_366\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/divc3xqtvwpiqtlvclyy\/PHID-FILE-u2dxf66pi2bo4tctgdkn\/335862-alphanumeric_aleo-white_F.png-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fkaelin\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6785559\" id=\"6785559\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fkaelin\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_113\"\u003efkaelin\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6785559\" data-sigil=\"has-tooltip\" data-meta=\"0_364\"\u003e\u003cspan class=\"screen-only\"\u003eJan 28 2021, 10:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-28 22:03:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_369\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6825675\" id=\"6825675\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003eJoe\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6825675\" data-sigil=\"has-tooltip\" data-meta=\"0_367\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 12 2021, 8:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-02-12 08:16:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_372\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nf444cwhprae6jyrfiz3\/PHID-FILE-ji22vh72t5pumvrskcp5\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ttaylor\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6886655\" id=\"6886655\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ttaylor\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_115\"\u003ettaylor\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6886655\" data-sigil=\"has-tooltip\" data-meta=\"0_370\"\u003e\u003cspan class=\"screen-only\"\u003eMar 5 2021, 3:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-05 15:27:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_375\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/giz4kiiwghycxmhpfwnm\/PHID-FILE-hoikbdz5d2t6fnwo7yxy\/bbbc81-alphanumeric_lato-dark_L.png-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/LSobanski\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6913674\" id=\"6913674\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/LSobanski\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003eLSobanski\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913674\" data-sigil=\"has-tooltip\" data-meta=\"0_373\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 1:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 13:30:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_388\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_384\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_385\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_386\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_387\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913730\" id=\"6913730\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003eMarostegui\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913730\" data-sigil=\"has-tooltip\" data-meta=\"0_382\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 1:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 13:45:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_380\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#3963629\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#3963629\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Milimetric\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Milimetric\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eWe still have to check Debezium with the DBAs and hear their thoughts on it, but it's possible we could go forward with both ways of generating events and figure out which is easier in practice:\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI've got some questions!\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhere is debezium supposed to run? If it needs to run on the actual primary masters to keep scanning for the binlogs I don't think I would feel confortable having a process that scans binlogs there (they are quite big) and adding more load to our masters. It doesn't feel good to have a process reading binlogs that are continuously used by that mysqld process. We should keep primary masters as clean as possible (we only run mysqld, pt-hearbteat and puppet there). They are critical enough :-)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhat happens with the PII that gets written to binlogs? (ie: passwords, emails)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhy not using this on clouddb1021? (the analytics clouddb replica) that already has sanitized version of our data, isn't as critical as primary masters and already receives sqoop processes?\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_397\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913765\" id=\"6913765\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_396\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_119\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913765\" data-sigil=\"has-tooltip\" data-meta=\"0_395\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 1:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 13:53:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_393\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_394\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_120\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWhere is debezium supposed to run?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIn k8s.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWe should keep primary masters as clean as possible\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgree, I'd prefer to consume the binlog of a replica.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWhat happens with the PII that gets written to binlogs?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe aren't going to produce MySQL all data as events.  I looked into this, and the output is just too raw and contextless to get the rich events we have currently.  My preferred solution is 'Change Data Capture via Transactional Outbox' one.  In that, we'd convert the EventBus extension to insert into one or several 'event outbox' tables in the exact format we want, likely just a JSON string that Debezium can hoist into an event.  So really, we don't even need the full binlog, just the logs for our outbox table(s).  (I think MySQL has a binlog table filter setting, right?)\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWhy not using this on clouddb1021\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis could work.  However, this would be in the critical path for production services like jobqueue and resource_purge.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_408\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_406\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_407\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913772\" id=\"6913772\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_405\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_121\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913772\" data-sigil=\"has-tooltip\" data-meta=\"0_404\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 1:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 13:55:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_402\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_403\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_122\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eReading the whole history here it seems that the problem we want to solve is a traditionally unsolvable one (keeping two logically-distinct datastores perfectly consistent in a distributed architecture while being not in a CP setup.\u003c\/p\u003e\n\n\u003cp\u003eWe should focus on reconciliation strategies instead of chasing for panaceas for problems we know cannot be solved reliably.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_419\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_417\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_418\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913777\" id=\"6913777\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_416\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_123\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913777\" data-sigil=\"has-tooltip\" data-meta=\"0_415\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 1:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 13:58:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_413\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_414\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_124\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6913765\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6913765\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eWhere is debezium supposed to run?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIn k8s.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWe should keep primary masters as clean as possible\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgree, I'd prefer to consume the binlog of a replica.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWhat happens with the PII that gets written to binlogs?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe aren't going to produce MySQL all data as events.  I looked into this, and the output is just too raw and contextless to get the rich events we have currently.  My preferred solution is 'Change Data Capture via Transactional Outbox' one.  In that, we'd convert the EventBus extension to insert into one or several 'event outbox' tables in the exact format we want, likely just a JSON string that Debezium can hoist into an event.  So really, we don't even need the full binlog, just the logs for our outbox table(s).  (I think MySQL has a binlog table filter setting, right?)\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWait, so the solution to eventgate being occasionally unreliable is to use a mysql table as an event log?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_429\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913793\" id=\"6913793\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6913793\" data-sigil=\"has-tooltip\" data-meta=\"0_427\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:04:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_425\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_126\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWait, so the solution to eventgate being occasionally unreliable is to use a mysql table as an event log?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIt isn't EventGate that is the problem here.  It's the lack of transactional guarantees between the event production and the state change in MySQL.  If we could or wanted to event source MediaWiki, this wouldn't be a problem, but that's kind of a non-starter I think :)\u003c\/p\u003e\n\n\u003cp\u003eI'd describe it more as using the existent MySQL replication 'event log' to carry the event\u00a0to Kafka.  If we use Debezium and the binlog, we won't be directly polling or selecting from the outbox table at all, just deleting everything from it after it is old (probably 7 days).\u003c\/p\u003e\n\n\u003cp\u003eAm very open to other ideas if you have them!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_438\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913810\" id=\"6913810\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_437\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_127\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913810\" data-sigil=\"has-tooltip\" data-meta=\"0_436\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:06:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_434\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_128\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWe should focus on reconciliation strategies instead of chasing for panaceas for problems we know cannot be solved reliably.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCan you explain why the binlog solution is not solving the problem reliably?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_441\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/yg5pg7brmam5e62oyhwa\/PHID-FILE-sjmc2yoi6v5p3takuaky\/25889f-alphanumeric_aleo-white_K.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Kormat\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6913813\" id=\"6913813\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Kormat\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_129\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Kormat\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913813\" data-sigil=\"has-tooltip\" data-meta=\"0_439\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:09:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_454\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_450\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_452\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913858\" id=\"6913858\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003eMarostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913858\" data-sigil=\"has-tooltip\" data-meta=\"0_448\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:26:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_446\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_447\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6913765\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6913765\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eWhere is debezium supposed to run?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIn k8s.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWe should keep primary masters as clean as possible\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgree, I'd prefer to consume the binlog of a replica.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThat's also something not easy for several reasons:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIt would need to run on all replicas, otherwise we'd have a SPOF and a snowflake. We already have too many snowflakes with some of the replicas (candidates masters, sanitarium masters, vslow...)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSome of the replicas have already quite a bunch of load and adding more reads latency isn't great (ie: enwiki)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eWhat happens with the PII that gets written to binlogs?\u003cbr \/\u003e\nSo really, we don't even need the full binlog, just the logs for our outbox table(s).  (I think MySQL has a binlog table filter setting, right?)\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eNot sure I am following - you mean for reading the binlog or for only replicating specific tables?\u003cbr \/\u003e\nFor reading, as far as I remember you only have a database filter. \u003cbr \/\u003e\nFor replicating, that means changing replication filters in production, which is, unfortunately a no-go as we cannot only replicate certain tables to our replicas. Everything needs to be replicated.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eWhy not using this on clouddb1021\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis could work.  However, this would be in the critical path for production services like jobqueue and resource_purge.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eBut on the other hand it is a separated environment that if there are any performance issues, it won't hit production.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_465\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_463\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_464\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913864\" id=\"6913864\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_462\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913864\" data-sigil=\"has-tooltip\" data-meta=\"0_461\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:28:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_459\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_460\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6913810\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6913810\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eWe should focus on reconciliation strategies instead of chasing for panaceas for problems we know cannot be solved reliably.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCan you explain why the binlog solution is not solving the problem reliably?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eBecause you will still have software failures, network partitions, error handling: this is basically the CAP theorem in action.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_475\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913869\" id=\"6913869\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_474\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6913869\" data-sigil=\"has-tooltip\" data-meta=\"0_473\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:32:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_471\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_472\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eBecause you will still have software failures, network partitions, error handling: this is basically the CAP theorem in action.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis is then true of the MySQL replicas then too, right?\u003c\/p\u003e\n\n\u003cp\u003eIIUC, the binlog has offsets.  Debezium acts like a regular MySQL replica reading the binlog, keeping track of where it is in the offsets.  If it fails, it should work just like a MySQL replica would: start from wherever it left off.\u003c\/p\u003e\n\n\u003cp\u003eI'm not claiming this would achieve full consistency always, just that we should be able to know when something is going wrong.  At the moment we have no way of knowing if e.g. every revision save results in a revision-create event.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_488\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_484\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_486\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_487\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6913886\" id=\"6913886\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_483\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003eMarostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913886\" data-sigil=\"has-tooltip\" data-meta=\"0_482\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:40:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_480\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_481\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6913869\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6913869\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eBecause you will still have software failures, network partitions, error handling: this is basically the CAP theorem in action.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis is then true of the MySQL replicas then too, right?\u003c\/p\u003e\n\n\u003cp\u003eIIUC, the binlog has offsets.  Debezium acts like a regular MySQL replica reading the binlog, keeping track of where it is in the offsets.  If it fails, it should work just like a MySQL replica would: start from wherever it left off.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eAssuming we have discarded the idea of replicate\/touch the master:\u003cbr \/\u003e\nHow would it handle if the replica goes down for a long time or even forever (ie HW crash)?\u003c\/p\u003e\n\n\u003cp\u003eRight now the way we handle that is...we just rebuilt it from either the backups\/another replica which are in 100% of the cases in a different binlog\/position. This can be mitigated if Debezium uses GTID, but in any case, this would be a problem.\u003cbr \/\u003e\nAnd as I mentioned above, it can be such an overhead to have another replica that is a snowflake cause it would have another replica hanging below it.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_491\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/yg5pg7brmam5e62oyhwa\/PHID-FILE-sjmc2yoi6v5p3takuaky\/25889f-alphanumeric_aleo-white_K.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Kormat\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6913900\" id=\"6913900\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_490\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Kormat\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Kormat\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_139\"\u003eDBA\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6913900\" data-sigil=\"has-tooltip\" data-meta=\"0_489\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 2:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 14:44:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_500\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914026\" id=\"6914026\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_499\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914026\" data-sigil=\"has-tooltip\" data-meta=\"0_498\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 3:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 15:20:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_496\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_497\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eAgree, I'd prefer to consume the binlog of a replica.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWhy not using this on clouddb1021\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAye, a clouddb1021 type setup makes a lot of sense for this.  Perhaps we could start with clouddb1021, and if we want something that is more 'production' we could set up another MySQL host with the same type of multi-instance replica setup.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIt would need to run on all replicas, otherwise we'd have a SPOF and a snowflake. \u003cbr \/\u003e\n...we just rebuilt it from either the backups\/another replica which are in 100% of the cases in a different binlog\/position\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAh I see good questions.  Needs more research I think, but there is GTID support in Debezium.  \u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html\u003c\/a\u003e  But I think I need to understand more about how we usually do replication setups.  I'm not sure I understand the 'snowflake' problem.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_509\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/yg5pg7brmam5e62oyhwa\/PHID-FILE-sjmc2yoi6v5p3takuaky\/25889f-alphanumeric_aleo-white_K.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Kormat\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914047\" id=\"6914047\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_508\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Kormat\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_142\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Kormat\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914047\" data-sigil=\"has-tooltip\" data-meta=\"0_507\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 3:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 15:25:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_505\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_506\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI had a quick look, as mariadb & mysql's GTID implementations are different and incompatible. Their roadmap says that they won't look at what's required to support mariadb until 1.6: \u003ca href=\"https:\/\/debezium.io\/roadmap\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/roadmap\/\u003c\/a\u003e. (The current stable version of Debezium is 1.4; 1.5 is the current development branch).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_518\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914058\" id=\"6914058\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_517\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_144\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914058\" data-sigil=\"has-tooltip\" data-meta=\"0_516\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 3:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 15:29:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_514\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_515\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_145\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eRelevant?\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-becomes-unavailable-while-debezium-is-running\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-becomes-unavailable-while-debezium-is-running\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_527\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914096\" id=\"6914096\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_526\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_146\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914096\" data-sigil=\"has-tooltip\" data-meta=\"0_525\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 3:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 15:32:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_523\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_524\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_147\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eTheir roadmap says that they won't look at what's required to support mariadb until 1.6\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAh interesting ok.\u003c\/p\u003e\n\n\u003cp\u003eEven if it were available, how would this work? Is GTIDs something we'd have to specifically enable on all MariaDB hosts?  If we did that, does that just make it possible for any replica to swap out which master it is replicating from (assuming that 'master' has all the relevant binlogs)?   (\/me is learning more about GTID replication...)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_537\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/yg5pg7brmam5e62oyhwa\/PHID-FILE-sjmc2yoi6v5p3takuaky\/25889f-alphanumeric_aleo-white_K.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Kormat\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914098\" id=\"6914098\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Kormat\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_148\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Kormat\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6914098\" data-sigil=\"has-tooltip\" data-meta=\"0_535\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 3:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 15:33:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_533\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_534\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_149\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6914058\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914058\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eRelevant?\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-becomes-unavailable-while-debezium-is-running\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-becomes-unavailable-while-debezium-is-running\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI don't think so; our mariadb clusters would count as a "Primary and replica" setup, not "High available clusters" (based on \u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#supported-mysql-topologies\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003esupported-mysql-topologies\u003c\/a\u003e).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_551\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_547\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_548\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_549\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_550\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914105\" id=\"6914105\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_546\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_150\"\u003eMarostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6914105\" data-sigil=\"has-tooltip\" data-meta=\"0_545\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 3:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 15:39:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_543\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_544\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_151\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6914026\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914026\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eAgree, I'd prefer to consume the binlog of a replica.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWhy not using this on clouddb1021\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAye, a clouddb1021 type setup makes a lot of sense for this.  Perhaps we could start with clouddb1021, and if we want something that is more 'production' we could set up another MySQL host with the same type of multi-instance replica setup.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eLet's keep that in mind for budgeting for next fiscal.\u003cbr \/\u003e\nHowever, having that host connected to production master is something that is operationally an overhead, as we'd need to include it under our failover strategies, as otherwise if we failover the master, that host will  keep replicating from the old one.\u003cbr \/\u003e\nIf it replicates from clouddb1021 that's already covered by our upstream master (the sanitarium host).\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eIt would need to run on all replicas, otherwise we'd have a SPOF and a snowflake. \u003cbr \/\u003e\n...we just rebuilt it from either the backups\/another replica which are in 100% of the cases in a different binlog\/position\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAh I see good questions.  Needs more research I think, but there is GTID support in Debezium.  \u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html\u003c\/a\u003e  But I think I need to understand more about how we usually do replication setups.  I'm not sure I understand the 'snowflake' problem.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThe snowflake problem is essentially that right now we have different types of hosts in production, they all run as replicas but have specific roles:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCandidate master (host that will take over in case of a master failure)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSanitarium master (host that has sanitarium host as a replica and then all the clouddb1021 hosts and runs ROW based replication)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003evslow,dump hosts: host that receive dump traffic, which sometimes cause load\/issues.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAll these create lots of operational burden on us, as we cannot simply exchange hosts if needed. We are in process of simplifying things to ease our operational load.\u003c\/p\u003e\n\n\u003cp\u003eIf we can have this specific connector hanging from clouddb1021, that already fixes helps us, as we'd need to worry about it, in terms of master failovers as there's likely nothing going to replace clouddb1021 :-)\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6914058\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914058\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eRelevant?\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-becomes-unavailable-while-debezium-is-running\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-becomes-unavailable-while-debezium-is-running\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eNot super relevant as the problem stands still, if we need to include this host in production then we do need to deal with possible master failures or swap.\u003cbr \/\u003e\nNone of our tools would be able to deal with it, and hence we'd need to manually change replication whenever there's a swap or a failure. \u003cbr \/\u003e\nIf it is connected to clouddb1021, then that's automatically handled by the upstream master, so any movement would be transparent to this host.\u003cbr \/\u003e\nObviously, if clouddb1021 crashes and it is not recoverable, then we do need to rebuild or think about how to handle it. A normal crash of clouddb1021 shouldn't be a big deal as replication will restart once it is back and will catch up once cloudb1021 has caught up with its upstream master.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_560\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914391\" id=\"6914391\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_559\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_152\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914391\" data-sigil=\"has-tooltip\" data-meta=\"0_558\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:19:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_556\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_557\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eInteresting, makes sense.\u003c\/p\u003e\n\n\u003cp\u003eA LooONng time ago when I did MySQL DBA work, to restore a replica, I would use LVM snapshots via \u003ca href=\"https:\/\/www.lenzg.net\/mylvmbackup\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003emylvmbackup\u003c\/a\u003e.  I'd have a dedicated replica on which I could lock tables, take a snapshot, and then unlock tables.  A background process would then mount the LVM snapshot and rsync the MySQL data directory elsewhere for backup or restore.   With this process, I believe the binlog offsets would be consistent between the original host and the restored replica.  I know this is different than how we do things, but in theory, would this work? Could we restore a crashed replica and then point Debezium (or any other MySQL replication consumer) at it, starting from the same binlog positions from the original?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_569\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914416\" id=\"6914416\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_568\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914416\" data-sigil=\"has-tooltip\" data-meta=\"0_567\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:23:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_565\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_155\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAh, if we needed to change the binlog position information used by Debezium, this is how: \u003ca href=\"https:\/\/debezium.io\/documentation\/faq\/#how_to_change_the_offsets_of_the_source_database\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/debezium.io\/documentation\/faq\/#how_to_change_the_offsets_of_the_source_database\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_579\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914434\" id=\"6914434\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_578\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_156\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6914434\" data-sigil=\"has-tooltip\" data-meta=\"0_577\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:25:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_575\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_576\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_157\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWe are in process of simplifying things to ease our operational load.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI'm interested in hearing more about this.  It sounds like the problems with using Debezium are similar to the problems of running MariaDB replicas! :)  Is part of this simplification effort to make it easier to change a replica's master?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_588\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/yg5pg7brmam5e62oyhwa\/PHID-FILE-sjmc2yoi6v5p3takuaky\/25889f-alphanumeric_aleo-white_K.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Kormat\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914481\" id=\"6914481\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_587\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Kormat\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_158\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Kormat\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914481\" data-sigil=\"has-tooltip\" data-meta=\"0_586\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:29:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_584\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_585\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_159\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFrom reading through their docs a bit:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDebezium requires \u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#enable-mysql-binlog\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebinlog_format=ROW\u003c\/a\u003e, which means it cannot connect directly to any of our masters (as we use STATEMENT).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIt also grabs a global read lock on the server it connects to when making an \u003ca href=\"https:\/\/debezium.io\/documentation\/reference\/connectors\/mysql.html#mysql-snapshots\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003einitial snapshot\u003c\/a\u003e which would also be a huge no-no against any pooled db instance. It's unspecified if it repeats this when making non-initial snapshots.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_597\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914499\" id=\"6914499\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_596\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_160\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914499\" data-sigil=\"has-tooltip\" data-meta=\"0_595\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:31:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_593\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_594\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eIt also grabs a global read lock on the server it connects to when making an initial snapshot which would also be a huge no-no against any pooled db instance. It's unspecified if it repeats this when making non-initial snapshots.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI don't know if this is configurable, but if it is, I don't think we need it for the Transactional Outbox idea.  In any case, I wouldn't want to run Debezium against a production db for sure.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_606\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914507\" id=\"6914507\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_605\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_162\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914507\" data-sigil=\"has-tooltip\" data-meta=\"0_604\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:31:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_602\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_603\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_163\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eDebezium requires binlog_format=ROW, which means it cannot connect directly to any of our masters (as we use STATEMENT).\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCould a dedicated replica for debezium (or on cloudb1021) turn on binlog using ROW format?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_617\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hhufjuud5kp5yiged4fs\/PHID-FILE-2p2bycgnb4gse5p3joee\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jcrespo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_615\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_616\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914516\" id=\"6914516\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_614\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jcrespo\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_165\"\u003ejcrespo\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914516\" data-sigil=\"has-tooltip\" data-meta=\"0_613\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:33:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_611\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_612\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_164\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e I am going to interject here, as backups owner.\u003c\/p\u003e\n\n\u003cp\u003eI have 2 needs regarding mediawiki databases and backups that I would like to solve, too. One is incremental backups for mediawiki dbs ("binlog backups"\/streaming backups), the other is incremental media backups (which requires a reliable way to get image updates).\u003c\/p\u003e\n\n\u003cp\u003eI would like, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e, if we could colaborate together towards a solution that works beyond analytics needs, so it could be useful also for backups and other teams\/needs.\u003c\/p\u003e\n\n\u003cp\u003eMany changes happen on the databases that do not touch the binlogs, as well as constant master switches, so not sure that is the right approach (schema changes, data consistency fixes, etc.). I also need (for backups) to do some consistency checks that may impact analytics data, but that is out of scope here.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_626\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914535\" id=\"6914535\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_625\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_166\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914535\" data-sigil=\"has-tooltip\" data-meta=\"0_624\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:38:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_622\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_623\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_167\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e+1 \u003ca href=\"\/p\/jcrespo\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jcrespo\u003c\/span\u003e\u003c\/a\u003e this ticket is about solving the MW event production consistency problem so we can use them as a 'source of truth' for things outside of MW\/MySQL.  But the backups and dumps problem is similar, as are other issues we will encounter will building event driven applications, e.g. how to bootstrap them with current state?  We should think of this as an org wide data integration problem and work on a common solution where we can :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_637\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_635\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_636\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914578\" id=\"6914578\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_634\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_168\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914578\" data-sigil=\"has-tooltip\" data-meta=\"0_633\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 4:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 16:47:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_631\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_632\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOk I'll try to re-summarize my argument:\u003cbr \/\u003e\nthe problem we're trying to solve is having transactional consistency between mediawiki and kafka. And we want to do it not at the application layer, but at the data layer, which is what I think is wrong for a few reasons. But before we go back to discussing solutions, I'd like to see a better explanation of the problem.\u003c\/p\u003e\n\n\u003cp\u003eThe problem statement I read here is:\u003c\/p\u003e\n\n\u003cp\u003e"We do have events produced by MediaWiki, but these events are decoupled from the MySQL writes, and there is no guarantee that e.g. every revision table save results in a mediawiki.revision-create event."\u003c\/p\u003e\n\n\u003cp\u003ewhich is a common reliability and logical consistency problem, but no analysis of the causes of such inconsistencies is provided. Once we have those spelled out, we can think of different approaches which do not involve using the database (and its binlog) as a message queue, and as an integration layer.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_646\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6914982\" id=\"6914982\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_645\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_170\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6914982\" data-sigil=\"has-tooltip\" data-meta=\"0_644\"\u003e\u003cspan class=\"screen-only\"\u003eMar 15 2021, 6:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-15 18:11:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_642\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_643\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eno analysis of the causes of such inconsistencies is provided.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eHm, I guess none is provided mostly because no consistency is expected with the current async logic (An \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/plugins\/gitiles\/mediawiki\/extensions\/EventBus\/+\/refs\/heads\/master\/includes\/EventBusHooks.php#336\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eHTTP POST from a MW deferred update\u003c\/a\u003e after the MySQL transaction is finished).\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI'd like to see a better explanation of the problem.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eMySQL is the source of truth for MediaWiki data.  We'd like to be able to rely on events as a source of truth instead of MW\/MySQL, so that we can make doing things like \u003ca href=\"\/T244590\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT244590: [Epic] Rework the WDQS updater as an event driven application\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e easier (and possible).  We want realtime-ish MW state changes without having to poll MySQL or the MW API for every new use case.  By putting those state changes as events in Kafka, we can build new services that consume the changes without being runtime coupled to MW.   However, if the MW events are not consistent, then we can't rely on them as a source of truth.\u003c\/p\u003e\n\n\u003cp\u003eIf the binlog reader is the problem here, we could of course just implement some custom MySQL poller process. This is the simple 'Transactional Outbox' solution. Ultimately this is how \u003ca href=\"https:\/\/www.confluent.io\/blog\/kafka-connect-deep-dive-jdbc-source-connector\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eKafka Connect JDBC\u003c\/a\u003e works too, but I don't think we want to (or can?) use that for the same reasons we don't really want the entire MediaWiki binlog as events (too low level).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_660\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/m25d4sm73mzvfxk7efxx\/PHID-FILE-zigk2s47ziyr7hqvpw7v\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ladsgroup\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_656\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_657\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_658\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_659\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6916250\" id=\"6916250\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_655\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ladsgroup\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_173\"\u003eLadsgroup\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6916250\" data-sigil=\"has-tooltip\" data-meta=\"0_654\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 2:10 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 02:10:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_652\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_653\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_172\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6914982\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914982\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eI'd like to see a better explanation of the problem.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eMySQL is the source of truth for MediaWiki data.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThat's a bit of oversimplification, MediaWiki does use for example s1 for enwiki but it's way more complex than that. For example, global users data comes from centralauth in s7, url shortener and some other mediawiki functionalities come from x1 (completely different hardware), texts of pages come for ES (yet another beast) and I assume we are not taking into account caching mediawiki has, like ParserCache, ...\u003c\/p\u003e\n\n\u003cp\u003eHaving mediawiki as an abstraction between analytics and databases is a good idea that ditching would cause all sorts of nastiness and edge cases you get handled for free in mw and the databases configuration and topology changes for reasons. It's like bypassing transport layer in OSI. If TCP is too slow, use UDP. Build something in mediawiki that's fast and quick instead of bypassing mediawiki altogether. You can take advantage of some caching even (depending on your usecase)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_673\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_669\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_670\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_671\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_672\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6917569\" id=\"6917569\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_668\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_174\"\u003eMarostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6917569\" data-sigil=\"has-tooltip\" data-meta=\"0_667\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 3:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 15:06:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_665\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_666\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_175\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6914507\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914507\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eDebezium requires binlog_format=ROW, which means it cannot connect directly to any of our masters (as we use STATEMENT).\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCould a dedicated replica for debezium (or on cloudb1021) turn on binlog using ROW format?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI would still go for clouddb1021, the concept of "dedicated" replica for this introduces yet another specific replica for a specific role, and we already have many of those (\u003ca href=\"\/T120242#6914105\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914105\u003c\/span\u003e\u003c\/a\u003e). clouddb1021 is owned by Analytics so we can set up ROW there if that's.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6914499\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6914499\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eIt also grabs a global read lock on the server it connects to when making an initial snapshot which would also be a huge no-no against any pooled db instance. It's unspecified if it repeats this when making non-initial snapshots.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI don't know if this is configurable, but if it is, I don't think we need it for the Transactional Outbox idea.  In any case, I wouldn't want to run Debezium against a production db for sure.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIf in the end, debezium is the solution you want to go for, I would prefer not to place another replica just for this for the reasons given above (keep in mind it would need to be a replica per section or a big host with multi-instance). If we can get it hang from clouddb1021 that it already has all the sections, that's better.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_683\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6917579\" id=\"6917579\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_682\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_176\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6917579\" data-sigil=\"has-tooltip\" data-meta=\"0_681\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 15:07:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_679\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_680\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHm, \u003ca href=\"\/p\/Ladsgroup\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ladsgroup\u003c\/span\u003e\u003c\/a\u003e I'm certainly not suggesting that we should ever bypass MediaWiki and get at MySQL at runtime.  What I mean is that if MediaWiki was totally shut down, and all 'caches' (including elasticsearch) were wiped, the 'source of truth' to turn everything back on would come from MySQL (+ media \/ wikitext content storage).\u003c\/p\u003e\n\n\u003cp\u003eOther services need consistent copies, just like caches do.  Having slightly inconsistent caches isn't such a big deal, but having e.g. inconsistent WDQS or incrementally updated MediaWiki history is a bit more of an issue.  This task is about making it possible to use events consistently and in realtime to \u003ca href=\"https:\/\/martinfowler.com\/articles\/201701-event-driven.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecarry state transfer\u003c\/a\u003e from MediaWiki elsewhere.\u003c\/p\u003e\n\n\u003cp\u003eIf MediaWiki could somehow offer transactional guarantees for events it emits, this wouldn't be an issue.  Until MW's MySQL transaction is closed, we don't consider MW state real.  By sourcing the events out of MySQL, we can be more confident about their consistency with MediaWiki's reality.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_686\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/bov64zymbxsy2emzxaut\/PHID-FILE-n37ejbchno4lp5hlru3f\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dcaro\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6917669\" id=\"6917669\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_685\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dcaro\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_178\"\u003edcaro\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6917669\" data-sigil=\"has-tooltip\" data-meta=\"0_684\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 3:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 15:31:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_695\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6917670\" id=\"6917670\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_694\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_179\"\u003eOttomata\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6917670\" data-sigil=\"has-tooltip\" data-meta=\"0_693\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 3:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 15:32:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_691\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_692\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eclouddb1021 is owned by Analytics so we can set up ROW there if that's\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCool sounds good, especially at first.  However, whatever the solution of this task is will likely be co-maintained by data engineering and platform engineering.  Many services (WDQS updater, job queue, change prop, analytics event ingestion, etc.) will rely on it, just like they rely on EventBus generated events right now.  In the future, I expect more and more event driven services to be built, all of which will mostly rely on events for state transfer, and many of them will want MediaWiki state.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_698\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tniasmrxjnwpefcxfpiy\/PHID-FILE-ctcb3luu4sewou7hdwlp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dcausse\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6918278\" id=\"6918278\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_697\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dcausse\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003edcausse\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6918278\" data-sigil=\"has-tooltip\" data-meta=\"0_696\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 4:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 16:56:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_709\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_707\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_708\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6918681\" id=\"6918681\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_706\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_183\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6918681\" data-sigil=\"has-tooltip\" data-meta=\"0_705\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 6:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 18:08:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_703\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_704\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_184\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6917670\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6917670\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eclouddb1021 is owned by Analytics so we can set up ROW there if that's\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eMany services (WDQS updater, job queue, change prop, analytics event ingestion, etc.) will rely on it, just like they rely on EventBus generated events right now.  In the future, I expect more and more event driven services to be built, all of which will mostly rely on events for state transfer, and many of them will want MediaWiki state.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThe solutions proposed here are not ok for me, and others have expressed disagreement. I think they will need further discussion (in the tech forum? with the interested\/impacted teams?) before implementation. I think this is a very dangerous path, and going against all best practices I can think of.\u003c\/p\u003e\n\n\u003cp\u003eWith this, I am NOT discarding the original problem statement ("we have several dependent datastores that can be derived from MediaWiki, let's find a reasonable way to jumpstart and update them reliably"). I would suggest a task describing the actual problem we want to solve is opened, instead of using this quite stale task which lists solutions first.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_719\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6919037\" id=\"6919037\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_718\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_185\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6919037\" data-sigil=\"has-tooltip\" data-meta=\"0_717\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 7:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 19:52:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_715\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_716\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_186\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eI think they will need further discussion (in the tech forum? with the interested\/impacted teams?) before implementation\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgree. I've asked Kate if she thinks this is tech forum worthy and she said that it probably was.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI would suggest a task describing the actual problem we want to solve is opened, instead of using this quite stale task which lists solutions first.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAm confused about this. I rewrote this task description within the past year to refocus the problem.  I think the task states the problem pretty clearly first, with possible solutions following.  Any other ideas would be very welcomed!  I think it's ok to discuss possible technical solutions before officially proposing them for implementation.\u003c\/p\u003e\n\n\u003cp\u003eIf by 'describing the actual problem' you mean more concrete use cases, I get your point.  I'll try to get together a concrete list of services that currently do rely on MediaWiki events, as well as ones in the near future that are likely to.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003egoing against all best practices I can think of\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e I'd be interesting hearing more about what best practices you are thinking of.  As for the possible solutions this task is exploring, I'm not making them up, but am gathering them from readings of 'best practices' out there.  In particular, I'd like to hear what is bad about these ideas.  Since we're talking about best practices, perhaps instead of focusing on these possible solutions, you could help me understand what is wrong with the ideas I am reading?  I linked the this \u003ca href=\"https:\/\/www.confluent.io\/blog\/turning-the-database-inside-out-with-apache-samza\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eTurning the database inside-out Strange Loop talk and blog post\u003c\/a\u003e in the task description, and I think its relevant here (minus the specifics about Samza).  I know I'm very steeped in event-driven world hype, but I think I can see event driven architectures as a way out of reimplementing the same one-off data pipelines over and over again.  But of course my real world experience with building them is very limited, and I'm very open to being wrong.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_730\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_728\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_729\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6919230\" id=\"6919230\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_727\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_187\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6919230\" data-sigil=\"has-tooltip\" data-meta=\"0_726\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 8:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 20:11:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_724\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_725\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_188\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe best practices I am talking about are, basically:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eDon't use the database as an integration layer\u003c\/strong\u003e. As \u003ca href=\"\/p\/Ladsgroup\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ladsgroup\u003c\/span\u003e\u003c\/a\u003e clearly pointed out, it turns ugly very fast once you consider real-life situations (like - an event likely needs to source from multiple MediaWiki datastores at once), and the advantage of "consistency" evaporates quickly\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cstrong\u003eDon't use the database as a queue\u003c\/strong\u003e. The proposal here is to insert events in the database (so making them part of the master db transaction) instead of producing them to a queue.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eWhat might not be clear from my opposition is that I am in favour of having an event-driven propagation of changes - we do so already, and I think doing it better is a plus. I just think this specific proposal is flawed.\u003c\/p\u003e\n\n\u003cp\u003eIMHO, it would be better to take a different approach, for instance (forgive the brevity, but I'm just trying to figure this out at 9 pm :)):\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eImprove the reliability and consistency of how events are produced in MediaWiki\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eImprove the reliability of the eventgate\/kafka conglomerate we use to ingest events from MediaWiki\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDevise a way to be able to link an event to the database transaction\/revid\/whatever that generated it, and have a system that allows reconciliation. For instance: one revision is missing from the event log but is present in the database? then generate the event that we're missing.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAlso, given one of our goals is to allow fast-loading data in new instances of dependent services:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eGenerate a "daily events snapshot" *from MediaWiki that allows fast-loading all of our data in a new dependent datastore by consuming that topic, then be able to consume the main events topic from the offset at which the snapshot was created\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThe reason for this last point is that consuming our whole events history to recreate state in a dependent datastore will not be fast nor efficient by any measure.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_739\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6919710\" id=\"6919710\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_738\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_189\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6919710\" data-sigil=\"has-tooltip\" data-meta=\"0_737\"\u003e\u003cspan class=\"screen-only\"\u003eMar 16 2021, 10:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-16 22:16:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_735\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_736\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_190\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks for responses, I want to respond more in full too, but here's a quick thought:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eDevise a way to be able to link an event to the database transaction\/revid\/whatever that generated it, and have a system that allows reconciliation. For instance: one revision is missing from the event log but is present in the database? then generate the event that we're missing.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI think this is similar to the transactional outbox idea.  There are many variations on this idea, but in the description I wrote:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eThe revision event is produced to Kafka. When the Kafka produce request succeeds, the revision_event_log's produced_at timestamp (or boolean) field is set.\u003cbr \/\u003e\nA separate process polls the revision_event_log table for records where produced_at is NULL, produces them to Kafka, and sets produced_at when the produce request succeeds.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI think your suggestion here is similar, but without the revision_event_log outbox table.  You are suggesting that we make it possible for a process to\u00a0poll MW tables and determine what events were not emitted, and then emit them.  I think that could work; although it might be difficult to recreate some of the context we have at original event time (e.g. \u003ctt class=\"remarkup-monospaced\"\u003euser_edit_count\u003c\/tt\u003e, \u003ctt class=\"remarkup-monospaced\"\u003epage_is_redirect\u003c\/tt\u003e), but I think it should be possible!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_742\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6920281\" id=\"6920281\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_741\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003eMarostegui\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_192\"\u003eTriage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_193\"\u003eIn progress\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003eDBA\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6920281\" data-sigil=\"has-tooltip\" data-meta=\"0_740\"\u003e\u003cspan class=\"screen-only\"\u003eMar 17 2021, 6:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-17 06:00:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_751\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6921342\" id=\"6921342\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_750\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_195\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6921342\" data-sigil=\"has-tooltip\" data-meta=\"0_749\"\u003e\u003cspan class=\"screen-only\"\u003eMar 17 2021, 1:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-17 13:12:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_747\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_748\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eGenerate a "daily events snapshot"\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgree that this will be needed for sure!  It isn't yet clear if we can do this 100% from Kafka, or will need a separate place (Hadoop? Swift?) to store this.  Having everything in Kafka would make bootstrapping applications simpler, but it might just be too big \/ inefficient as you say.  Maybe Kafka \u003ca href=\"https:\/\/towardsdatascience.com\/log-compacted-topics-in-apache-kafka-b1aa1e4665a7\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecompacted topics\u003c\/a\u003e will help us?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eDon't use the database as a queue\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI'd argue that neither CDC (consuming the binlog) or the Outbox ideas are doing this.  Perhaps this is just terminology, but what they are doing is using the database as a temporary log.\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCDC just relies on the existent replication binlog (which...I wouldn't call a queue?).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI'd also call the Outbox table a log (similar to the the MW logging table), but I guess it depends on how it is used.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIn the Hybrid CDC + Outbox idea, nothing ever reads the Outbox table.  Its content is consumed from the replication binlog and produced into Kafka as soon as possible.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIn the normal Outbox (no CDC), there is a separate process that polls the Outbox and produces the events.  If, at request time, we write to the Outbox, produce the event, and then mark the Outbox record as produced (outside of the transaction), the Outbox becomes a simple temporary buffer for which the poller is only looking for records that have yet to be produced.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cblockquote\u003e\u003cp\u003eDon't use the database as an integration layer. As \u003ca href=\"\/p\/Ladsgroup\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ladsgroup\u003c\/span\u003e\u003c\/a\u003e clearly pointed out, it turns ugly very fast once you consider real-life situations (like - an event likely needs to source from multiple MediaWiki datastores at once), and the advantage of "consistency" evaporates quickly\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI don't fully understand this one, could you explain more?  How is using the database to get consistency between database state and events going to affect the way events are used?  We already produce events in a deferred update over HTTP outside of a transaction, so there is no consistency between multiple databases.  MediaWiki itself has to deal with multiple database consistency, right?  In the cases where the event needs to source from multiple MediaWiki datastores, the events will have the same consistency problem as MediaWiki does.  If we utilize database transactions for this, we can at least be guaranteed that events will be consistent with at least ONE MediaWiki data store, which is really the best that MediaWiki can do anyway (right?).\u003c\/p\u003e\n\n\u003chr class=\"remarkup-hr\" \/\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e, given the difficulties in using binlog CDC in our current MariaDB setup (I'm going to try to start saying MariaDB instead of MySQL) outlined by Manuel and Stevie, I'm starting to more prefer a polling based option.  I think using an Outbox is still a good idea though, mainly because it gives us full control over how the events are modeled, and would be much easier to code and integrate with EventBus now.  What about this?\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDuring the DB transaction, EventBus writes the event into an outbox table with a \u003ctt class=\"remarkup-monospaced\"\u003epayload\u003c\/tt\u003e and a \u003ctt class=\"remarkup-monospaced\"\u003eproduced_at\u003c\/tt\u003e DEFAULT NULL timestamp (this will mostly be used as a boolean).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIn the deferred update, EventBus POSTs the event to EventGate, and if successful, sets the \u003ctt class=\"remarkup-monospaced\"\u003eproduced_at\u003c\/tt\u003e field.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eA separate poller process selects\u00a0records from the outbox table where \u003ctt class=\"remarkup-monospaced\"\u003eproduced_at IS NULL\u003c\/tt\u003e, and POSTs the \u003ctt class=\"remarkup-monospaced\"\u003epayload\u003c\/tt\u003e to EventGate, and if successful, sets the \u003ctt class=\"remarkup-monospaced\"\u003eproduced_at\u003c\/tt\u003e field.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThis or another process deletes records where \u003ctt class=\"remarkup-monospaced\"\u003eproduced_at IS NOT NULL\u003c\/tt\u003e.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eOr some variation.  This is pretty similar to your reconciliation idea, but allows EventBus to continue working as is, with a little eventual-consistency-reconciliation added in.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_761\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6925040\" id=\"6925040\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_760\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_197\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6925040\" data-sigil=\"has-tooltip\" data-meta=\"0_759\"\u003e\u003cspan class=\"screen-only\"\u003eMar 18 2021, 2:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-18 14:27:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_757\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_758\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eHow would it handle if the replica goes down for a long time or even forever (ie HW crash)?\u003cbr \/\u003e\nRight now the way we handle that is...we just rebuilt it from either the backups\/another replica which are in 100% of the cases in a different binlog\/position. \u003cbr \/\u003e\n...\u003cbr \/\u003e\nNot super relevant as the problem stands still, if we need to include this host in production then we do need to deal with possible master failures or swap.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e Am trying to understand this a little more.  How do you do master swaps now?  From these comments  I understood that the binlog position was not in sync in all the different binlogs out there.  It that correct?  I.e.\u003c\/p\u003e\n\n\u003cp\u003emaster A -> replica B -> replica C\u003c\/p\u003e\n\n\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003einsert into revision ...\u003c\/tt\u003e (rev_id 10) might be at position 456 for master A, but at position 123 in the binlog that replicas from B -> C.\u003c\/p\u003e\n\n\u003cp\u003eDid I get that right?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_772\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6941567\" id=\"6941567\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_769\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_199\"\u003eOttomata\u003c\/a\u003e renamed this task from \u003cspan class=\"phui-timeline-value\"\u003eReliable (atomic) MediaWiki event production \/ MediaWiki events as source of truth\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eConsistent MediaWiki state change events | MediaWiki events as source of truth\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6941567\" data-sigil=\"has-tooltip\" data-meta=\"0_768\"\u003e\u003cspan class=\"screen-only\"\u003eMar 24 2021, 2:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-24 14:38:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_770\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_200\"\u003eOttomata\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-be4esxb4vomrrbi\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_771\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_201\"\u003eOttomata\u003c\/a\u003e added subscribers: \u003ca href=\"\/p\/Protsack.stephan\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_202\"\u003eProtsack.stephan\u003c\/a\u003e, \u003ca href=\"\/p\/Zbyszko\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_203\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Zbyszko\u003c\/a\u003e.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_766\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_767\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_204\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eI'll try to get together a concrete list of services that currently do rely on MediaWiki events, as well as ones in the near future that are likely to.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI started this process by asking some folks I know want to rely on MediaWiki state change events to describe what they needed it for, and I edited the description of the task to include their comments.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_781\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6945584\" id=\"6945584\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_780\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_205\"\u003eOttomata\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6945584\" data-sigil=\"has-tooltip\" data-meta=\"0_779\"\u003e\u003cspan class=\"screen-only\"\u003eMar 25 2021, 3:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-25 15:19:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_777\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_778\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_206\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnother idea that may not be feasible:  Would it be possible to move the event produce call out of the deferred update to before MediaWiki closes the MariaDB transaction?  I.e.\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eopen MariaDB transaction\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003einsert into revision, etc.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eproduce event\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eclose MariaDB transaction\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eIn this way, we might produce spurious events which are not actually persisted in the database, but that should be easier to reconcile than missing events.  E.g. Most consumers would probably  have to ask MW API later for the revision content anyway, and if MW doesn't have the event's rev_id, MW API request will let the consumer know.\u003c\/p\u003e\n\n\u003cp\u003eI'm not sure what happens with the rev_ids in our MariaDB transactions though. Are they created and available to MediaWiki before the transaction is closed?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_791\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kd4xv6kmiqlmbg5i3ynf\/PHID-FILE-q756a47n7hkxmwqviotl\/profile-profile.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aaron\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6947570\" id=\"6947570\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_790\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_207\"\u003eaaron\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6947570\" data-sigil=\"has-tooltip\" data-meta=\"0_789\"\u003e\u003cspan class=\"screen-only\"\u003eMar 25 2021, 11:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-25 23:23:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_787\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_788\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_208\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eKeep in mind that, strictly speaking, some of these problems are not even solved in MediaWiki for "core" DB shards. These include the "main" s[1-8] shards and the "extension" x1 shard. For example, a web request might update an \u003ca href=\"\/S1\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eS1\u003c\/span\u003e\u003c\/a\u003e (enwiki) and S7 (centralauth) in one "transaction round", which just means that each of the relevant DB connections are checked for connectivity (pinged if there was no activity < 1 sec ago), and, after that passes, then the COMMITs are made in rapid succession. It is still possible, though very unlikely, that a proper subset of the transactions fail. Also, some events might be triggered from onTransaction() callbacks or PRESEND deferred updates.\u003c\/p\u003e\n\n\u003cp\u003eLuckily, the non-core "external store" DB servers are not an issue, since we uses AUTOCOMMIT mode for the blob insertions, failure to find somewhere to store a blob aborts the main transaction round (of core DB shards), and orphaned external store blobs don't matter. Also, we can atomically commit to multiple core databases on the same shard\/master (e.g. metawiki and centralauth), so that is not an issue either.\u003c\/p\u003e\n\n\u003cp\u003eSo...back to MediaWiki events... Some events might be "higher level" than "something happened in this DB", e.g. they may or may not be coupled to writes to other core DBs. Also, those writes will mostly be in the transaction round, but some might happen shortly after it. If only one of the DB's COMMITs succeeded (rare, but possible), do you say that the whole event failed or not?\u003c\/p\u003e\n\n\u003cp\u003eAnother complexity is around the metadata used in kafka events to figure out what was committed or not. DBAs can change the masters for a shard and even move DBs to different shards (e.g. moving wikidata from s5 to s8).  This is tracking in "sectionsByDB" in wmf-config. The "extension" shards are more complicated. Suppose you have 5 extensions participate in a transaction that involves kafka events, the DBA makes x1 read-only and moves 1 extension off to x3. Ideally, we'd use a "sectionsByModule" LBFactory option like "sectionsByDB", but no such thing exists (just various per-extension config vars). Even with that, any "queue\/log in a DB" approach would be impractical.\u003c\/p\u003e\n\n\u003cp\u003eAnyway, suppose that we *are* just talking about events that only involve writes to a single core DB server. In theory, we could do something like:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eBEGIN\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e...issue regular writes queires...\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003egenerate a monotonic "global commit id" UUID\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eINSERT a row with the UUID into \u003ctt class=\"remarkup-monospaced\"\u003egcids_committed\u003c\/tt\u003e (each MW core mysql server would have *one* such table, not one per database)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003esynchronously push JSON events to kafka, each including the UUID and DB shard; if the kafka response is not definitive success (including "non-answers"), then ROLLBACK and abort\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCOMMIT\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eKafka might have phantom events (e.g. not committed in the DB), but it would never be missing events. Kafka consumers would have to distinguish two types of phantom events: events from failed transactions, which can be discarded, and events that belong to pending transactions, which cannot be treated as actionable nor discarded. If the transaction is pending, the consumer should wait until the transaction ends before processing the event or any newer events.\u003c\/p\u003e\n\n\u003cp\u003eA kafka consumer could distinguish pending and failed transaction by using READ_UNCOMMITTED and LOCK_IN_SHARE_MODE. You can test this locally using a sql client, even 'maintenance\/sql.php' (assuming MediaWiki uses mysql):\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: CREATE TABLE IF NOT EXISTS gcids_committed (gcid DECIMAL(29));\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: BEGIN;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: INSERT INTO gcids_committed (gcid) VALUES (1);\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: SELECT * FROM gcids_committed WHERE gcid=1 LOCK IN SHARE MODE;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: (now blocked waiting terminal A transaction)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: SELECT * FROM gcids_committed WHERE gcid=1 LOCK IN SHARE MODE;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: (now blocked waiting terminal A transaction)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: COMMIT;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: (now unblocked)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: (now unblocked)\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eFor postgres, using READ UNCOMMITTED would not work; one could instead use pg_advisory_lock\/pg_advisory_lock_shared. A similar approach for mysql would be using GET_LOCK(), though the kafka consumers would block each other a bit. E.g.:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: CREATE TABLE IF NOT EXISTS gcids_committed (gcid DECIMAL(29));\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: BEGIN;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: SELECT GET_LOCK('gcid-id-1',0) -- ROLLBACK if this yields 0 -- ;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: INSERT INTO gcids_committed (gcid) VALUES (1);\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: SELECT GET_LOCK('gcid-id-1',4294967295);\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: (now blocked waiting terminal A transaction)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: SELECT GET_LOCK('gcid-id-1',4294967295);\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: (now blocked waiting terminal A transaction)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: COMMIT;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal A: SELECT RELEASE_LOCK('gcid-id-1');\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: (now unblocked)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: SELECT * FROM gcids_committed WHERE gcid=1;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal B: SELECT RELEASE_LOCK('gcid-id-1');\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: (now unblocked)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: SELECT * FROM gcids_committed WHERE gcid=1;\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTerminal C: SELECT RELEASE_LOCK('gcid-id-1');\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eIn terms of sudden master crashes and replica promotion, we do use semi-sync replication to increase the odds that no data is lost. On the enwiki master, mysql has rpl_semi_sync_master_wait_point = AFTER_COMMIT, so the GET_LOCK approach would be safer, since kafka consumers would not see gcids_committed UUIDs only known to the master (in semi-sync replication mode). In any case, it is possible for semi-sync replication to automatically stop waiting on replicas for while due to rpl_semi_sync_master_timeout, so there is no hard safety guarantee here.\u003c\/p\u003e\n\n\u003cp\u003eNote that the above scheme does not involve pushing bulky JSON blobs to mysql. It would add some read traffic to the shard masters though (1 query per event per subscribed consumer). It was also require synchronous kafka writes, though one could batch write them during the pre-COMMIT stage.  The gcids_committed tables would be one-per-server and append-only. Shards would not be removed unless consumers are all caught up (we've only removed es* shards before AFAIK, and those don't matter, so this should be a minor point). If a wiki moves from main shard A to B, then the DBA would leave gcids_committed alone on shard A (even though they would eventually prune the old copy of the wiki DB). This makes it easy on consumers trying to validate kafka events. If an extension is moved from extension shard 1 to 3, the same approach would be used.\u003c\/p\u003e\n\n\u003cp\u003eSince we *do* have transaction rounds spanning several core\/extension DB servers, you'd either have to pick the most "meaningful" transaction (usually the one from the connection to the wiki's main shard server) or store them all and check all of them (though it is not obvious what to do if only some failed).\u003c\/p\u003e\n\n\u003cp\u003eAs \u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e said, some kind of reconciliation approach would be good. Regardless of what "probabilistic" atomicity improvements are made, it will probably be necessary as a fallback when something goes wrong.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_794\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gaj2wbcq54jfzcscgqrn\/PHID-FILE-jyedy4j2gqeydwbnujxx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ottomata\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7020067\" id=\"7020067\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_793\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ottomata\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_209\"\u003eOttomata\u003c\/a\u003e mentioned this in \u003ca href=\"\/T249745\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_210\"\u003eT249745: Could not enqueue jobs: "Unable to deliver all events: 503: Service Unavailable"\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7020067\" data-sigil=\"has-tooltip\" data-meta=\"0_792\"\u003e\u003cspan class=\"screen-only\"\u003eApr 20 2021, 1:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-04-20 13:12:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_797\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/giz4kiiwghycxmhpfwnm\/PHID-FILE-hoikbdz5d2t6fnwo7yxy\/bbbc81-alphanumeric_lato-dark_L.png-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/LSobanski\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7038723\" id=\"7038723\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_796\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/LSobanski\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_211\"\u003eLSobanski\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_212\"\u003eIn progress\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1060\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_213\"\u003ePending comment\u003c\/a\u003e on the \u003ca href=\"\/tag\/dba\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_214\"\u003eDBA\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7038723\" data-sigil=\"has-tooltip\" data-meta=\"0_795\"\u003e\u003cspan class=\"screen-only\"\u003eApr 27 2021, 3:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-04-27 15:57:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_810\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_806\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_807\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_808\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_809\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7069452\" id=\"7069452\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_805\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_215\"\u003eMarostegui\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7069452\" data-sigil=\"has-tooltip\" data-meta=\"0_804\"\u003e\u003cspan class=\"screen-only\"\u003eMay 7 2021, 1:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-07 13:52:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_802\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_803\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_216\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T120242#6925040\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120242#6925040\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Ottomata\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Ottomata\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eHow would it handle if the replica goes down for a long time or even forever (ie HW crash)?\u003cbr \/\u003e\nRight now the way we handle that is...we just rebuilt it from either the backups\/another replica which are in 100% of the cases in a different binlog\/position. \u003cbr \/\u003e\n...\u003cbr \/\u003e\nNot super relevant as the problem stands still, if we need to include this host in production then we do need to deal with possible master failures or swap.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Marostegui\u003c\/span\u003e\u003c\/a\u003e Am trying to understand this a little more.  How do you do master swaps now?  From these comments  I understood that the binlog position was not in sync in all the different binlogs out there.  It that correct?  I.e.\u003c\/p\u003e\n\n\u003cp\u003emaster A -> replica B -> replica C\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eThe process is as follows:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWe move all the slaves under the future master (which is still a replica from the current master). The process to do that is basically to stop replication at the same time on both hosts for a few seconds, once they are stopped (on the same binlog position) we move one of the hosts under the future master. That is repeated with all the slaves in the topology.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOnce we have master > future master > all the replicas we set read-only on the master so the binlog stops moving.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOnce that is done, we move the current master under the future master and disconnect replicationthere, so the future master has now no master on top, becoming the master of the topology and the old master becomes a replica.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFinally, we change MW to point to the new master\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWe remove read-only\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-d2tiurktk2jgxzcpd2jw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gedvamxugwvij32kcdtw","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ioco7rvk5qrecmohnyak","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ioco7rvk5qrecmohnyak","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vbyvvtbztxaeuaxelxx4","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-cav5zgwjbayv33seltga"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vbyvvtbztxaeuaxelxx4","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-i6rmwmg6stvi6vxfdelt"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ctchvhisqfvk4rz7nt7d","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ctchvhisqfvk4rz7nt7d","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ctchvhisqfvk4rz7nt7d","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-SPCE-6l6g5p53yi3mypnlpxjw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg","contextPHID":"PHID-TASK-jvhyszid3tb7h5dkgwcz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vbyvvtbztxaeuaxelxx4"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-hhn7svjs5isgzkbexbia"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ganhiwt4ew3pxrmmdpca"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-7nmqeeo6pfndicguymjd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gpb2ynmxbgmy4ld2555w"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-lwdtp6ibyfb5lhpnr3ve"}},{"hovercardSpec":{"objectPHID":"PHID-USER-gpb2ynmxbgmy4ld2555w"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-lwdtp6ibyfb5lhpnr3ve"}},{"hovercardSpec":{"objectPHID":"PHID-USER-u7w6n5ecde66oujx33pe"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-7ihdeqzhrbpqho67o7gc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-g2u3mzw5hq5yluuoozey"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-qon3ihsksxdvat4jvph3"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-vtxgzcwqt5xtc6fh3rkx"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-hpmkoa4j6capez3imhrb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-nfqkx6pruggu3y4"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-p3j6ssmfws7xnf6"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-2b7oz62ylk3jk4aus262"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-2dh7qysleg7hqnj"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-g7bfwahewstu6un6ewf6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-g7bfwahewstu6un6ewf6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vbyvvtbztxaeuaxelxx4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ioco7rvk5qrecmohnyak"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-smg6zxxehxrd7jhpap4o"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-2b7oz62ylk3jk4aus262"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-ppipgoogxo6byzc"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vbyvvtbztxaeuaxelxx4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ioco7rvk5qrecmohnyak"}},{"phid":"PHID-XACT-TASK-kp6nrcapfkz6pv6"},{"hovercardSpec":{"objectPHID":"PHID-USER-ioco7rvk5qrecmohnyak"}},{"phid":"PHID-XACT-TASK-pcavshjwdk366e7"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-bcnjn3oeepyob2f"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"phid":"PHID-XACT-TASK-e2bxdrw4foghunm"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-vbyvvtbztxaeuaxelxx4"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-hpmkoa4j6capez3imhrb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-q2inkkqhmsawczthflaq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-3ojbpvdjrw3kfhkmlbua"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"hovercardSpec":{"objectPHID":"PHID-USER-x2vchxmpgcslynvavhgz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-eo4cqx5xwnf4ojlk4bbl"}},{"phid":"PHID-XACT-TASK-cwwzok27zvhh4e2"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-jthrwxtavin4rmt"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-u4xmxbh3phvlfwm"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-6emelmmnhr5ta2v"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-hsl5kfcfcd3fptg"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-odpwxnogjtjy5mr"},{"hovercardSpec":{"objectPHID":"PHID-USER-ftzpbliicrvqlkvlqy2b"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-ooje44wwlve52pd"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-dukz2jkq3pxahhz"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-u2g2ynljasm3dht"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-xobwlv7vxdlxvxu"},{"hovercardSpec":{"objectPHID":"PHID-USER-ftzpbliicrvqlkvlqy2b"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-qyhw2ngdk6p2oac"},{"hovercardSpec":{"objectPHID":"PHID-USER-ftzpbliicrvqlkvlqy2b"}},{"phid":"PHID-XACT-TASK-l6a3jpevowhowvp"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-xjsdkuy7heortlv"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-s5bbnwgfnbplkky"},{"hovercardSpec":{"objectPHID":"PHID-USER-ftzpbliicrvqlkvlqy2b"}},{"phid":"PHID-XACT-TASK-65a4cxmvetddhtc"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-3k7p6qswd43tr5f"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-r2hwgcsdiligske"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-mr4ia5iysszappc"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-uyflycwo3upo6qu"},{"hovercardSpec":{"objectPHID":"PHID-USER-ftzpbliicrvqlkvlqy2b"}},{"phid":"PHID-XACT-TASK-6mmnl4emyreac3m"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-gkk3kncgrlolbyr"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-23imq3hck4vmxyf"},{"phid":"PHID-XACT-TASK-2xskjf6472vxroa"},{"hovercardSpec":{"objectPHID":"PHID-USER-gzd4fguky5ixnvz46fvu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-kluwjxjgw666edd"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-gvf4ytnef3kz4qf"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-ynmqzeuzxtn6o4t"},{"phid":"PHID-XACT-TASK-5c4f4vmr2ddrptp"},{"hovercardSpec":{"objectPHID":"PHID-USER-ctchvhisqfvk4rz7nt7d"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-jhubbfeopescped"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-w5tyvultkjkfbby"},{"hovercardSpec":{"objectPHID":"PHID-USER-vvww3faq62q2b7j2nfxb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-ufpqiae2dceyipb"},{"hovercardSpec":{"objectPHID":"PHID-USER-ume3zcaj4r5xel5ohu56"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-vfmwks345bqnaml"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-v522nodfzzohiku"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-2cgpgaj7ghuegh3"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-a5y4du7zdxgj773"},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-bk6sgdzdhctxupi3vn5b"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-5iuvn4kjilwynldgts56"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-xsesr7tshdzctej"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-3uvfcgrr6ylzapw"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tqxkrxqt7eyhtouaapnc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6amic3xetvajxpt6tzmq"}},{"phid":"PHID-XACT-TASK-nglu75ztebpnqu6"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"phid":"PHID-XACT-TASK-pccvt57jjaaqlqf"},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"phid":"PHID-XACT-TASK-rqjccqelfnkq5dk"},{"hovercardSpec":{"objectPHID":"PHID-USER-tafngdco2cilcyr7qhhg"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-amtg54o2xkeyh7dg2rws"}},{"hovercardSpec":{"objectPHID":"PHID-USER-eo4cqx5xwnf4ojlk4bbl"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-5iuvn4kjilwynldgts56"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-jaiwk5qqweitcc3dlad7"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-hwibeuyzizzy4xzunfsk"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-uqxxlc3n6qnfohe"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nhruhahftzko2fp","anchor":"4397775"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4cxno35zxxreps5","anchor":"4400363"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-6osdfeggemofvwk","anchor":"4474702"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yesbhynouquh7yk","anchor":"4518712"},{"tip":"Via Bulk Update"},[],{"phid":"PHID-XACT-TASK-7o5g4ie3z77d6bw","anchor":"4837384"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sk5oqicpwwsr6aq","anchor":"6054962"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-s2hsh23rtznfhyh","anchor":"6055221"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4gfc6esbpbuhaky","anchor":"6471024"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nfqkx6pruggu3y4\/","ref":"T120242#6471045"},[],{"anchor":"6471045"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_242\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nfqkx6pruggu3y4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_244\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-nfqkx6pruggu3y4\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nfqkx6pruggu3y4","anchor":"6471045"},{"tip":"Via Web"},[],[],[],[],{"phid":"PHID-XACT-TASK-wsywz4iudqyfe7z","anchor":"6669518"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-p3j6ssmfws7xnf6\/","ref":"T120242#6669528"},[],{"anchor":"6669528"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_258\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_259\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-p3j6ssmfws7xnf6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_260\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_261\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-p3j6ssmfws7xnf6","anchor":"6669528"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-isnmvjci4udp2vl","anchor":"6669543"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2dh7qysleg7hqnj\/","ref":"T120242#6669547"},[],{"anchor":"6669547"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_271\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_272\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2dh7qysleg7hqnj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_273\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2dh7qysleg7hqnj","anchor":"6669547"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cgl23kxhvjjjejd","anchor":"6669548"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-kvle6dbrlmldw6h","anchor":"6672474"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-ybcpihx5z3peuzm","anchor":"6673764"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3cp2j34c22fcabt","anchor":"6677516"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ppipgoogxo6byzc\/","ref":"T120242#6678059"},[],{"anchor":"6678059"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_294\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ppipgoogxo6byzc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_296\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_297\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ppipgoogxo6byzc","anchor":"6678059"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pkljqre3z75ypqc","anchor":"6699804"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-livgdrtet2zenva","anchor":"6710348"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dl5lhwjog6fknfn","anchor":"6710691"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ovx4grepqa4ugut","anchor":"6722700"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kp6nrcapfkz6pv6\/","ref":"T120242#6741183"},[],{"anchor":"6741183"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_316\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kp6nrcapfkz6pv6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_318\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-c4cgw7v7rqmhry4","anchor":"6741183"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pcavshjwdk366e7\/","ref":"T120242#6741415"},[],{"anchor":"6741415"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_325\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pcavshjwdk366e7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pcavshjwdk366e7","anchor":"6741415"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bcnjn3oeepyob2f\/","ref":"T120242#6741422"},[],{"anchor":"6741422"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_334\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_335\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bcnjn3oeepyob2f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_336\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bcnjn3oeepyob2f","anchor":"6741422"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-e2bxdrw4foghunm\/","ref":"T120242#6763163"},[],{"anchor":"6763163"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_343\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_344\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-e2bxdrw4foghunm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_345\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-iwnvakowtky3prs","anchor":"6763163"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wb55zvj6zpmg63g","anchor":"6765826"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jqweo4nnn6s6m6u","anchor":"6765832"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nfdafvwcfokum3q","anchor":"6766371"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sfkwqrmip7psc3o","anchor":"6785438"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-a2fbtcoj7xsvnnc","anchor":"6785559"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xtto2otgvitxbgf","anchor":"6825675"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-65q6khxafnmffea","anchor":"6886655"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5522y4n7xoxfrbq","anchor":"6913674"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cwwzok27zvhh4e2\/","ref":"T120242#6913730"},[],{"anchor":"6913730"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_376\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cwwzok27zvhh4e2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_378\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-cwwzok27zvhh4e2","anchor":"6913730"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jthrwxtavin4rmt\/","ref":"T120242#6913765"},[],{"anchor":"6913765"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_389\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jthrwxtavin4rmt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_391\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jthrwxtavin4rmt","anchor":"6913765"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u4xmxbh3phvlfwm\/","ref":"T120242#6913772"},[],{"anchor":"6913772"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_398\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u4xmxbh3phvlfwm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_400\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-u4xmxbh3phvlfwm","anchor":"6913772"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6emelmmnhr5ta2v\/","ref":"T120242#6913777"},[],{"anchor":"6913777"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_409\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_410\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6emelmmnhr5ta2v\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_411\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_412\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-6emelmmnhr5ta2v","anchor":"6913777"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hsl5kfcfcd3fptg\/","ref":"T120242#6913793"},[],{"anchor":"6913793"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_420\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_421\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hsl5kfcfcd3fptg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_422\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_423\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-hsl5kfcfcd3fptg\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hsl5kfcfcd3fptg","anchor":"6913793"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-odpwxnogjtjy5mr\/","ref":"T120242#6913810"},[],{"anchor":"6913810"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_430\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-odpwxnogjtjy5mr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_432\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-odpwxnogjtjy5mr","anchor":"6913810"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ddd27tlmkkvmm26","anchor":"6913813"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ooje44wwlve52pd\/","ref":"T120242#6913858"},[],{"anchor":"6913858"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_442\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_443\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ooje44wwlve52pd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_444\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_445\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ooje44wwlve52pd","anchor":"6913858"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dukz2jkq3pxahhz\/","ref":"T120242#6913864"},[],{"anchor":"6913864"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_455\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_456\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dukz2jkq3pxahhz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_457\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-dukz2jkq3pxahhz","anchor":"6913864"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u2g2ynljasm3dht\/","ref":"T120242#6913869"},[],{"anchor":"6913869"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_466\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_467\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u2g2ynljasm3dht\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_468\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_469\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-u2g2ynljasm3dht\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_470\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u2g2ynljasm3dht","anchor":"6913869"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xobwlv7vxdlxvxu\/","ref":"T120242#6913886"},[],{"anchor":"6913886"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_476\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_477\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xobwlv7vxdlxvxu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_478\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_479\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-xobwlv7vxdlxvxu","anchor":"6913886"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-punpn4o3gneqshq","anchor":"6913900"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qyhw2ngdk6p2oac\/","ref":"T120242#6914026"},[],{"anchor":"6914026"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_492\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_493\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qyhw2ngdk6p2oac\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_494\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_495\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qyhw2ngdk6p2oac","anchor":"6914026"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-l6a3jpevowhowvp\/","ref":"T120242#6914047"},[],{"anchor":"6914047"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_501\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_502\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l6a3jpevowhowvp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_503\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_504\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-l6a3jpevowhowvp","anchor":"6914047"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xjsdkuy7heortlv\/","ref":"T120242#6914058"},[],{"anchor":"6914058"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_510\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_511\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xjsdkuy7heortlv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_512\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_513\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xjsdkuy7heortlv","anchor":"6914058"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-s5bbnwgfnbplkky\/","ref":"T120242#6914096"},[],{"anchor":"6914096"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_519\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_520\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-s5bbnwgfnbplkky\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_521\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_522\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-s5bbnwgfnbplkky","anchor":"6914096"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-65a4cxmvetddhtc\/","ref":"T120242#6914098"},[],{"anchor":"6914098"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_528\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_529\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-65a4cxmvetddhtc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_530\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_531\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-65a4cxmvetddhtc\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_532\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-65a4cxmvetddhtc","anchor":"6914098"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3k7p6qswd43tr5f\/","ref":"T120242#6914105"},[],{"anchor":"6914105"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_538\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_539\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3k7p6qswd43tr5f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_540\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_541\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3k7p6qswd43tr5f\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_542\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3k7p6qswd43tr5f","anchor":"6914105"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-r2hwgcsdiligske\/","ref":"T120242#6914391"},[],{"anchor":"6914391"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_552\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_553\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r2hwgcsdiligske\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_554\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_555\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-r2hwgcsdiligske","anchor":"6914391"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mr4ia5iysszappc\/","ref":"T120242#6914416"},[],{"anchor":"6914416"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_561\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_562\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mr4ia5iysszappc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_563\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mr4ia5iysszappc","anchor":"6914416"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uyflycwo3upo6qu\/","ref":"T120242#6914434"},[],{"anchor":"6914434"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_570\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_571\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uyflycwo3upo6qu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_572\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-uyflycwo3upo6qu\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_574\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uyflycwo3upo6qu","anchor":"6914434"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6mmnl4emyreac3m\/","ref":"T120242#6914481"},[],{"anchor":"6914481"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_580\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_581\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6mmnl4emyreac3m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_582\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_583\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6mmnl4emyreac3m","anchor":"6914481"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gkk3kncgrlolbyr\/","ref":"T120242#6914499"},[],{"anchor":"6914499"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_589\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_590\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gkk3kncgrlolbyr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_591\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_592\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gkk3kncgrlolbyr","anchor":"6914499"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-23imq3hck4vmxyf\/","ref":"T120242#6914507"},[],{"anchor":"6914507"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_598\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_599\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-23imq3hck4vmxyf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_600\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_601\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-23imq3hck4vmxyf","anchor":"6914507"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2xskjf6472vxroa\/","ref":"T120242#6914516"},[],{"anchor":"6914516"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_607\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_608\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2xskjf6472vxroa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_609\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_610\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-2xskjf6472vxroa","anchor":"6914516"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kluwjxjgw666edd\/","ref":"T120242#6914535"},[],{"anchor":"6914535"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_618\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_619\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kluwjxjgw666edd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_620\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_621\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kluwjxjgw666edd","anchor":"6914535"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gvf4ytnef3kz4qf\/","ref":"T120242#6914578"},[],{"anchor":"6914578"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_627\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_628\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gvf4ytnef3kz4qf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_629\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_630\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-gvf4ytnef3kz4qf","anchor":"6914578"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ynmqzeuzxtn6o4t\/","ref":"T120242#6914982"},[],{"anchor":"6914982"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_638\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_639\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ynmqzeuzxtn6o4t\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_640\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_641\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ynmqzeuzxtn6o4t","anchor":"6914982"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5c4f4vmr2ddrptp\/","ref":"T120242#6916250"},[],{"anchor":"6916250"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_647\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_648\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5c4f4vmr2ddrptp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_649\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_650\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-5c4f4vmr2ddrptp\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_651\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-5c4f4vmr2ddrptp","anchor":"6916250"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jhubbfeopescped\/","ref":"T120242#6917569"},[],{"anchor":"6917569"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_661\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_662\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jhubbfeopescped\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_663\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_664\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-jhubbfeopescped","anchor":"6917569"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-w5tyvultkjkfbby\/","ref":"T120242#6917579"},[],{"anchor":"6917579"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_674\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_675\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-w5tyvultkjkfbby\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_676\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_677\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-w5tyvultkjkfbby\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-w5tyvultkjkfbby","anchor":"6917579"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-y7xohjm4cbivyju","anchor":"6917669"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ufpqiae2dceyipb\/","ref":"T120242#6917670"},[],{"anchor":"6917670"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_687\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_688\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ufpqiae2dceyipb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_689\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_690\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-g3ztejf2z2ihih6","anchor":"6917670"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pmjy7nki6nxkm3g","anchor":"6918278"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vfmwks345bqnaml\/","ref":"T120242#6918681"},[],{"anchor":"6918681"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_699\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_700\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vfmwks345bqnaml\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_701\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_702\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-vfmwks345bqnaml","anchor":"6918681"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-v522nodfzzohiku\/","ref":"T120242#6919037"},[],{"anchor":"6919037"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_710\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_711\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v522nodfzzohiku\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_712\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_713\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-v522nodfzzohiku\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-v522nodfzzohiku","anchor":"6919037"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2cgpgaj7ghuegh3\/","ref":"T120242#6919230"},[],{"anchor":"6919230"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_720\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_721\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2cgpgaj7ghuegh3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_722\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_723\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-2cgpgaj7ghuegh3","anchor":"6919230"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-a5y4du7zdxgj773\/","ref":"T120242#6919710"},[],{"anchor":"6919710"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_731\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_732\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-a5y4du7zdxgj773\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_733\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_734\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-a5y4du7zdxgj773","anchor":"6919710"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qeojxb4kobnsatk","anchor":"6920281"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xsesr7tshdzctej\/","ref":"T120242#6921342"},[],{"anchor":"6921342"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_743\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_744\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xsesr7tshdzctej\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_745\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_746\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xsesr7tshdzctej","anchor":"6921342"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3uvfcgrr6ylzapw\/","ref":"T120242#6925040"},[],{"anchor":"6925040"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_752\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_753\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3uvfcgrr6ylzapw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_754\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_755\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3uvfcgrr6ylzapw\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_756\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3uvfcgrr6ylzapw","anchor":"6925040"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nglu75ztebpnqu6\/","ref":"T120242#6941567"},[],{"anchor":"6941567"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_762\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_763\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nglu75ztebpnqu6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_764\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_765\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],[],{"phid":"PHID-XACT-TASK-tihfmjb4tpwoj4k","anchor":"6941567"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pccvt57jjaaqlqf\/","ref":"T120242#6945584"},[],{"anchor":"6945584"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_773\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_774\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pccvt57jjaaqlqf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_775\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_776\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pccvt57jjaaqlqf","anchor":"6945584"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rqjccqelfnkq5dk\/","ref":"T120242#6947570"},[],{"anchor":"6947570"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_782\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_783\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rqjccqelfnkq5dk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_784\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_785\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-rqjccqelfnkq5dk\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_786\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rqjccqelfnkq5dk","anchor":"6947570"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z5bz6aek67cgt36","anchor":"7020067"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6ujpgtlipksweog","anchor":"7038723"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uqxxlc3n6qnfohe\/","ref":"T120242#7069452"},[],{"anchor":"7069452"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_798\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_799\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uqxxlc3n6qnfohe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_800\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_801\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-uqxxlc3n6qnfohe","anchor":"7069452"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2f0e5413\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}