for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_26\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2967073\" id=\"2967073\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_25\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2967073\" data-sigil=\"has-tooltip\" data-meta=\"0_24\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2017, 10:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-24 22:37:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_29\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2967088\" id=\"2967088\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_28\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_6\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/1674651\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2967088\" data-sigil=\"has-tooltip\" data-meta=\"0_27\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2017, 10:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-24 22:37:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_32\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2967346\" id=\"2967346\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_31\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-wnmgd2r6yyqlm4h\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2967346\" data-sigil=\"has-tooltip\" data-meta=\"0_30\"\u003e\u003cspan class=\"screen-only\"\u003eJan 24 2017, 11:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-24 23:56:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_35\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2967391\" id=\"2967391\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_34\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-24yry5hfikkxech\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2967391\" data-sigil=\"has-tooltip\" data-meta=\"0_33\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2017, 12:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-25 00:22:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_38\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2967414\" id=\"2967414\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_37\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-ajxj2whxkjdpt2g\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2967414\" data-sigil=\"has-tooltip\" data-meta=\"0_36\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2017, 12:36 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-25 00:36:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_42\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2967543\" id=\"2967543\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_40\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-rzvnf5fdybd4til\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2967543\" data-sigil=\"has-tooltip\" data-meta=\"0_39\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2017, 1:20 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-25 01:20:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_41\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-ojo3kbjrlednvrw\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2970308\" id=\"2970308\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_50\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2970308\" data-sigil=\"has-tooltip\" data-meta=\"0_49\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2017, 8:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-25 20:47:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_47\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_48\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_14\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\n\n\u003cblockquote\u003e\u003cp\u003e\u003cstrong\u003eDesign changes from current schema\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSwitch to "latest hash" revision retention policy, which keeps only one one render of the latest revision. While the current implementation is already quite close to what we need, there are some changes we would need to make to this policy:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eUse TTLs instead of plain deletions whenever grace_ttl is > 0.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe regular "latest hash" policy still keeps outdated renders for a TTL, to guarantee that edits can complete. There is however still a chance that edits based on an almost-expired entry would fail due to expiry. To avoid this, we can consider adding some "TTL extension" logic, which would re-write such content on read access to make sure that edits based on TTLed content will complete. This could be driven by a \u003ctt class=\"remarkup-monospaced\"\u003egrace_ttl_renew_threshold\u003c\/tt\u003e configuration variable, which would trigger rewrites with a new TTL lease for items that have less TTL than this value remaining. To communicate whether something is a bona fide stored item or a TTLed one, TTLed responses could expose their remaining lifetime in a header (cache-control, for example). Higher level modules like parsoid could then use this to request corresponding content (like data-parsoid) when needed to ensure its TTL extension, before returning old HTML. As a result of all this, edits based on old revisions still in TTLed storage are also guaranteed to finish.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOptimize the compaction strategy for the new update pattern. With the small dataset (<500G total) and different update pattern, TWCS might not be the best choice.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003e\u003cstrong\u003eMigration strategy\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEnable content versioning filter: Re-renders all content that is not up to spec.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTruncate html \/ data-parsoid tables\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTruncate HTML first, to avoid data-parsoid loss issues. However, loss of original HTML would still be an issue for selective serialization.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCurrently, pretty sure we return errors when original HTML is missing, which means that outstanding errors could not be saved.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eTemporarily change this behavior to ignore loss of original HTML & accept a few dirty diffs. Number of edits affected likely small.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAlternative: Disable VE during transition. Would have to be for longer than most edits, though (hours?).\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEnable "latest hash" revision retention policy with TTL extension logic.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAdjust compaction strategy.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCollect data on performance over next months. Should give us decent idea of "current" table performance.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI don't have strong feelings about truncating the parsoid tables, except for a concern that it sets an expectation that the OOMs will cease; I don't think there is any reason to believe that either of \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T153588\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eT153588\u003c\/a\u003e or \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T156155\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eT156155\u003c\/a\u003e could have been avoided by truncating the tables.  These were the result of accumulated tombstones within a single revision, something we'd continue to be vulnerable to even if we're only retaining the current revisions, (possibly more so given the other proposed changes, latest_hash, TTL renewal, etc).\u003c\/p\u003e\n\n\u003cp\u003eFinally, assuming that \u003cem\u003e"Optimize the compaction strategy for the new update pattern"\u003c\/em\u003e is a reference to LCS (as was mentioned in our team meeting), that also concerns me because the issues with both revision retention policies and partition width in general are inextricably linked to tombstone GC, and leveled compaction is notoriously bad in this respect.  Of course, if this wasn't a reference to LCS, I'd be interested to hear what you have in mind here.\u003c\/p\u003e\n\n\u003cp\u003eTo put these concerns into context, it's my understanding that you mean for this to be a short-term solution, something more or less immediately actionable to stabilize things while we work on a longer vision covered by \u003ca href=\"\/T120171\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT120171: RFC: Differentiate storage strategies for archival storage vs. hot current data\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e.  I worry that this approach trades a set of knowns for more unknowns, without any assurances that it won't make things worse.  I'd rather that we spend the time to build on everything we've learned, consult with other teams as necessary, and to invest heavily in testing and validation to get this right once and for all.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2970595\" id=\"2970595\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_60\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2970595\" data-sigil=\"has-tooltip\" data-meta=\"0_59\"\u003e\u003cspan class=\"screen-only\"\u003eJan 25 2017, 10:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-25 22:04:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_57\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_58\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_16\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eFinally, assuming that "Optimize the compaction strategy for the new update pattern" is a reference to LCS\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI intentionally left the details of this open. It would be great if you could make a recommendation.\u003c\/p\u003e\n\n\u003cp\u003eI think we agree that a smaller dataset will simplify the problem, and improve performance and reliability. Unless there is some reasonably easy way to conclusively test whether it will solve all OOMs as well, I would propose to set things up as best as we can, and then see if there is indeed still a problem to solve.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_70\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2972925\" id=\"2972925\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_69\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2972925\" data-sigil=\"has-tooltip\" data-meta=\"0_68\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2017, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-26 15:58:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_66\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_67\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_18\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T156199#2970595\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT156199#2970595\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eFinally, assuming that "Optimize the compaction strategy for the new update pattern" is a reference to LCS\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI intentionally left the details of this open. It would be great if you could make a recommendation.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWhat we have now is TWCS + bi-weekly user-defined compactions that collapse older windows into one another.  This seems far from ideal from an implementation perspective, but the results at least are quite good, so I think it pays to be a bit more conservative here.  My recommendation would be to continue with the plan for Cassandra 3.x testing, as 3.2 introduced \u003ca href=\"https:\/\/issues.apache.org\/jira\/browse\/CASSANDRA-6696\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eCASSANDRA-6696\u003c\/a\u003e, allowing us to carve up the compaction space to increase locality in the same way we foresaw very high instance counts doing (think ScyllaDB).\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI think we agree that a smaller dataset will simplify the problem, and improve performance and reliability.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI think we agree that a smaller dataset \u003cem\u003ewon't hurt\u003c\/em\u003e.  On that front, my only concern there is that if we're trading away our historical data in exchange for an expectation of fewer\/no OOMs, \u003ca href=\"https:\/\/en.wikipedia.org\/wiki\/Buyer's_remorse\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebuyer's remorse\u003c\/a\u003e could be a problem.  If you're OK with that, so am I.\u003c\/p\u003e\n\n\u003cp\u003eI'm more concerned about the other changes, the ones meant to retain only the current revision.  The revision retention policy is a major problem of the current design, and these proposed changes will result in the generation of even more tombstones.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eUnless there is some reasonably easy way to conclusively test whether it will solve all OOMs as well, I would propose to set things up as best as we can, and then see if there is indeed still a problem to solve.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe could create an alternate table that uses the proposed retention policy, and double-write to \u003cem\u003etry-and-see\u003c\/em\u003e, I'd just rather spend the time researching a solution that eliminates the revision retention policies as we know them.  Because if doing so is truly intractable, then we should probably be reconsidering our choice of storage technology.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_80\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2973062\" id=\"2973062\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2973062\" data-sigil=\"has-tooltip\" data-meta=\"0_78\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2017, 4:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-26 16:29:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_76\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_77\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_20\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWhat we have now is TWCS + bi-weekly user-defined compactions that collapse older windows into one another. This seems far from ideal from an implementation perspective, but the results at least are quite good\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSo you are saying that TWCS + custom compactions would still make sense for a table that has only current revisions?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eWe could create an alternate table that uses the proposed retention policy, and double-write to try-and-see, I'd just rather spend the time researching a solution that eliminates the revision retention policies as we know them. Because if doing so is truly intractable, then we should probably be reconsidering our choice of storage technology.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIt's clearly not intractable, given enough effort. However, the other solutions we have discussed for current revision storage are significantly more complex and expensive. We should thus be rather sure that we'd actually need them, and we should also consider alternatives to cassandra if we really need to go down that route.\u003c\/p\u003e\n\n\u003cp\u003eSo, if there is a practical way to test Cassandra behavior with a reasonable number of renders per article and day (assuming rate limiting on re-renders), then that would be great. I believe one of the recent edge cases was a gallery page on commons with 14mb of HTML. If we can re-write that kind of content ~24 times per day over a couple of days, and then not get a OOM on read, then chances are quite good that we won't need a major rewrite & increase in complexity.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/sql6zxypsdgnvcij2wjr\/PHID-FILE-pgi2xyqugx6rvnlyptyo\/648d72-alphanumeric_lato-dark_P.png-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Pchelolo\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2983939\" id=\"2983939\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Pchelolo\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Pchelolo\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2983939\" data-sigil=\"has-tooltip\" data-meta=\"0_87\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2017, 11:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-30 23:03:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_85\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_86\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_22\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've created a document capturing an idea of a completely different solution: \u003ca href=\"https:\/\/docs.google.com\/document\/d\/1qd8XilG5Jt0TRm5mMEokCG6d0_DkyReCi85KKlh-i8c\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/docs.google.com\/document\/d\/1qd8XilG5Jt0TRm5mMEokCG6d0_DkyReCi85KKlh-i8c\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eLet's keep it in gdocs for now since it's easier to comment on sections of the document there and it's still in a very vague stage.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2983940\" id=\"2983940\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_91\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-gnxooh27tagtdbz\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2983940\" data-sigil=\"has-tooltip\" data-meta=\"0_90\"\u003e\u003cspan class=\"screen-only\"\u003eJan 30 2017, 11:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-01-30 23:04:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-uqe3lpykg4r5qawkkepq"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-7rjnewhk74dfkxjye5fu"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-uqe3lpykg4r5qawkkepq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-uqe3lpykg4r5qawkkepq"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-rcfwx2ps4hytfs2"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-vmhfmbvfwrmypq3"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-63jyirsf63tygjg"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-eeq2dxh637uqvgj"},{"hovercardSpec":{"objectPHID":"PHID-USER-gpb2ynmxbgmy4ld2555w"}},{"phid":"PHID-XACT-TASK-sjzc65quufxwhdy"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4zq2bfekggx3p2i","anchor":"2967073"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-ym2z2yyfp5tdyl7","anchor":"2967088"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wnmgd2r6yyqlm4h","anchor":"2967346"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-24yry5hfikkxech","anchor":"2967391"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ajxj2whxkjdpt2g","anchor":"2967414"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-rzvnf5fdybd4til","anchor":"2967543"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rcfwx2ps4hytfs2\/","ref":"T156199#2970308"},[],{"anchor":"2970308"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_43\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_44\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rcfwx2ps4hytfs2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_45\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_46\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rcfwx2ps4hytfs2","anchor":"2970308"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vmhfmbvfwrmypq3\/","ref":"T156199#2970595"},[],{"anchor":"2970595"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_52\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_53\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vmhfmbvfwrmypq3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_54\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_55\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-vmhfmbvfwrmypq3\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_56\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vmhfmbvfwrmypq3","anchor":"2970595"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-63jyirsf63tygjg\/","ref":"T156199#2972925"},[],{"anchor":"2972925"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_62\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_63\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-63jyirsf63tygjg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_64\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_65\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-63jyirsf63tygjg","anchor":"2972925"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-eeq2dxh637uqvgj\/","ref":"T156199#2973062"},[],{"anchor":"2973062"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_71\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_72\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eeq2dxh637uqvgj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_73\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_74\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-eeq2dxh637uqvgj\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_75\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eeq2dxh637uqvgj","anchor":"2973062"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-sjzc65quufxwhdy\/","ref":"T156199#2983939"},[],{"anchor":"2983939"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_81\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_82\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sjzc65quufxwhdy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_83\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sjzc65quufxwhdy","anchor":"2983939"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gnxooh27tagtdbz","anchor":"2983940"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/45b62efe\/core.pkg.css"]}