for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-acxz2a7hhyfbbq6z7clz\/?after=1410256&quoteTargetID=UQ0_5&quoteRef=T95253\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_248\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1410256\" id=\"1410256\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_1\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Matanya\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003eMatanya\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/794934\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1410256\" data-sigil=\"has-tooltip\" data-meta=\"0_246\"\u003e\u003cspan class=\"screen-only\"\u003eJun 29 2015, 5:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-06-29 17:28:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_251\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1410258\" id=\"1410258\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_250\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/833\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/833\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eUnder discussion\u003c\/a\u003e on the \u003ca href=\"\/tag\/restbase\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003eRESTBase\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1410258\" data-sigil=\"has-tooltip\" data-meta=\"0_249\"\u003e\u003cspan class=\"screen-only\"\u003eJun 29 2015, 5:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-06-29 17:28:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_260\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1452113\" id=\"1452113\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_259\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1452113\" data-sigil=\"has-tooltip\" data-meta=\"0_258\"\u003e\u003cspan class=\"screen-only\"\u003eJul 14 2015, 4:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-14 16:57:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_256\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have done some testing with ipv6 docker containers on hosts with a \/64 subnet assigned to them: \u003ca href=\"https:\/\/github.com\/wikimedia\/ansible-deploy\/tree\/master\/roles\/docker_debian\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/wikimedia\/ansible-deploy\/tree\/master\/roles\/docker_debian\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eBy using the existing \/64 subnets, it avoids the need to manually assign additional ipv4 aliases. However, routing for ipv6 was still manual when I tested this, so each node needed an explicit route for each of the \/64 subnets.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_265\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1452125\" id=\"1452125\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e renamed this task from \u003cspan class=\"phui-timeline-value\"\u003eSet up multiple Cassandra instances per hardware node\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eTest multiple Cassandra instances per hardware node\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1452125\" data-sigil=\"has-tooltip\" data-meta=\"0_261\"\u003e\u003cspan class=\"screen-only\"\u003eJul 14 2015, 4:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-14 16:58:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_263\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eBBlack\u003c\/a\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eMoritzMuehlenhoff\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_268\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1452134\" id=\"1452134\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-z4x2togqd4uepoq\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1452134\" data-sigil=\"has-tooltip\" data-meta=\"0_266\"\u003e\u003cspan class=\"screen-only\"\u003eJul 14 2015, 5:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-14 17:03:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_277\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oczrgb2xczu3hebwulsk\/PHID-FILE-3raolhvprjakga3u36ek\/alphanumeric_lato-white_B.png-_5880b0-255%2C255%2C255%2C0.7.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BBlack\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1453721\" id=\"1453721\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003eBBlack\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1453721\" data-sigil=\"has-tooltip\" data-meta=\"0_275\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2015, 1:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-15 13:12:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_273\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI had to backtrack a bit into the ref'd closed ticket, but it seems that the fundamental drivers for multiple instances per HW node is that:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf there's too much stuff in a single instance, GC perf goes down the drain (long pauses)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf we size the hw nodes smaller such that a single instance per hw is small enough for GC to cope with, the total cost of the cluster increases unacceptably because more smaller nodes costs more than fewer larger nodes\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eIs that a correct understanding of the arguments here?  I think I'd still rather see us use more small nodes and bite the cost as a natural consequence of the GC perf, or fix the GC perf issue.  Putting multiple instances on a single hw node sounds ugly.  Each instance is going to assume it owns the (virtual or not) machine's resources, and systemd\/docker isn't going to be sufficient to contain all performance-related side-channel interaction between them in terms of i\/o, network, vm subsystem, cpu\/cache, etc.\u003c\/p\u003e\n\n\u003cp\u003eIs spiky\/freezy GC with large datasets inherent to Cassandra regardless of what's done with the underlying Java?  Can it be addressed with a different JVM, a different GC algorithm, etc?  I'm no expert on Java+GC issues, but I generally tend to think this is a huge, common flaw in the whole Java ecosystem.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_287\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1454051\" id=\"1454051\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_286\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1454051\" data-sigil=\"has-tooltip\" data-meta=\"0_285\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2015, 3:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-15 15:03:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_283\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_284\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@BBlack\u003c\/span\u003e\u003c\/a\u003e: GC scaling is an important and well-known issue for the JVM (even with G1GC) and more generally any GC'd language runtime. It's partly driven by laws of physics, so it's unlikely that we'll be able to make any quick progress there. Many have tried and keep trying. However, it looks like it's not the only issue for Cassandra. There might also be timeouts and race conditions triggered by the larger data size playing into this. Large instances are not recommended and rarely used, and thus more likely to be buggy.\u003c\/p\u003e\n\n\u003cp\u003eI don't see a good reason for discarding moderate isolation out of hand. Even multiple processes improve isolation between requests *a lot* over a single VM (the current status quo). We could get pretty far in cross-request isolation if we went for *extremely* small instances (<100G?) on shared hardware. However, in practice that is not the only priority; it's not even the top one for C*. I think a handful of moderately-sized instances (<600G\/instance) could be a good compromise between throughput, cost and isolation in the longer term. We are running multiple services on these and many other hardware nodes already, and I don't see us moving back to one-hardware-node-per-process at all.\u003c\/p\u003e\n\n\u003cp\u003eThe cost implications at stake at this scale aren't completely trivial. 53T per cluster divided into hw nodes with 500G mean load each would make for 106 very small nodes, plus the labor required to rack & image those, switch ports, pay for DC space, cooling & power. This is per DC, so multiply by two. We'd have to find some significant extra budget to pay for this.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_296\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1454385\" id=\"1454385\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1454385\" data-sigil=\"has-tooltip\" data-meta=\"0_294\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2015, 4:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-15 16:36:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_292\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_293\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1453721\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1453721\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/BBlack\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@BBlack\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI had to backtrack a bit into the ref'd closed ticket, but it seems that the fundamental drivers for multiple instances per HW node is that:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf there's too much stuff in a single instance, GC perf goes down the drain (long pauses)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIf we size the hw nodes smaller such that a single instance per hw is small enough for GC to cope with, the total cost of the cluster increases unacceptably because more smaller nodes costs more than fewer larger nodes\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eIs that a correct understanding of the arguments here?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYes, that pretty much sums it up.  When an instance size reaches a certain point (where size is some function of the dataset size and transaction throughput), heap usage makes GC problematic.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI think I'd still rather see us use more small nodes and bite the cost as a natural consequence of the GC perf, or fix the GC perf issue.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIt's not so much that GC is broken per say (the JVMs GC implementations are pretty awesome, in fact).  It's more that this use-case is pretty much the pathological worst-case scenario for a garbage collector, combined with a need for very low\/consistent request latencies (which STW pauses wreak havoc on).\u003c\/p\u003e\n\n\u003cp\u003eIncremental progress on this front is made all the time, and practical Cassandra node densities have increased a lot over the past releases, but this is a long game.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003ePutting multiple instances on a single hw node sounds ugly.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI agree.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eEach instance is going to assume it owns the (virtual or not) machine's resources, and systemd\/docker isn't going to be sufficient to contain all performance-related side-channel interaction between them in terms of i\/o, network, vm subsystem, cpu\/cache, etc.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eInstance isolation is one concern, storage density of single hosts is another; Operationally there are things that have to be figured out to do this.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIs spiky\/freezy GC with large datasets inherent to Cassandra regardless of what's done with the underlying Java?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eFor some value of spiky\/freezy, yes.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eCan it be addressed with a different JVM, a different GC algorithm, etc?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe recently moved to G1GC, which seems to be a pretty significant leap over CMS in that it achieves better results with less tuning (though it requires more headroom to work its 'magic').  It's not a game changer, though, we need to run smaller node densities for the foreseeable future.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI'm no expert on Java+GC issues, but I generally tend to think this is a huge, common flaw in the whole Java ecosystem.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgain, not that it changes things, but in this case I think it's more about the use of GC (any GC) for this type of use-case.  I know of no other GC'd language\/runtime that would fare as well.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_299\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1454432\" id=\"1454432\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-vccb7acddgagker\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1454432\" data-sigil=\"has-tooltip\" data-meta=\"0_297\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2015, 4:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-15 16:50:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_302\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1454533\" id=\"1454533\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-rakziltd7aheemy\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1454533\" data-sigil=\"has-tooltip\" data-meta=\"0_300\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2015, 5:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-15 17:03:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_311\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1465308\" id=\"1465308\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1465308\" data-sigil=\"has-tooltip\" data-meta=\"0_309\"\u003e\u003cspan class=\"screen-only\"\u003eJul 20 2015, 4:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-20 16:00:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_307\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFrom an IRC conversation today with Moritz regarding the use of firejail for this purpose:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003e11:30 <moritzm> if not sure if it's the most viable option, the focus of firejail is on isolation from a security perspective. but I doubt its  possibilities to configure resource distribution\/consumption are sufficient for your use case\u003cbr \/\u003e\n11:30 <moritzm> e.g. what happens if one of the compartmented nodes is highly loaded\u003cbr \/\u003e\n11:31 <moritzm> to ensure that it doesn't harm the others\u003cbr \/\u003e\n11:32 <moritzm> firejail is designed to  ensure that if a firejailed process is compromised, it's unable to attack other bits of the system\u003cbr \/\u003e\n11:33 <moritzm> also, you'll probably need to run all nodes on the same port for Cassandra?\u003cbr \/\u003e\n11:33 <urandom> they'll each need their own IP, yeah\u003cbr \/\u003e\n11:33 <moritzm> going with systemd nspawn containers or a full-blown virtualisation seems more appropriate to me\u003cbr \/\u003e\n11:34 <moritzm> maybe you can experiment with one of the nodes in our new ganeti cluster\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_320\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1465354\" id=\"1465354\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1465354\" data-sigil=\"has-tooltip\" data-meta=\"0_318\"\u003e\u003cspan class=\"screen-only\"\u003eJul 20 2015, 4:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-20 16:16:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_316\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1465308\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1465308\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eFrom an IRC conversation today with Moritz regarding the use of firejail for this purpose:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003e11:30 <moritzm> if not sure if it's the most viable option, the focus of firejail is on isolation from a security perspective. but I doubt its  possibilities to configure resource distribution\/consumption are sufficient for your use case\u003cbr \/\u003e\n11:30 <moritzm> e.g. what happens if one of the compartmented nodes is highly loaded\u003cbr \/\u003e\n11:31 <moritzm> to ensure that it doesn't harm the others\u003cbr \/\u003e\n11:32 <moritzm> firejail is designed to  ensure that if a firejailed process is compromised, it's unable to attack other bits of the system\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eAFAIK, not even container-based solutions can fully help in this edge case.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003e11:33 <moritzm> also, you'll probably need to run all nodes on the same port for Cassandra?\u003cbr \/\u003e\n11:33 <urandom> they'll each need their own IP, yeah\u003cbr \/\u003e\n11:33 <moritzm> going with systemd nspawn containers or a full-blown virtualisation seems more appropriate to me\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003e+1 for systemd-nspawn.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003e11:34 <moritzm> maybe you can experiment with one of the nodes in our new ganeti cluster\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eNot sure. AFAIK, the ganeti cluster is meant for non-public-facing services. But yes, I agree we need to experiment somewhere \/ somehow.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_330\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1465372\" id=\"1465372\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_329\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1465372\" data-sigil=\"has-tooltip\" data-meta=\"0_328\"\u003e\u003cspan class=\"screen-only\"\u003eJul 20 2015, 4:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-20 16:21:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_326\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_327\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1465354\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1465354\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/mobrovac\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@mobrovac\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1465308\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1465308\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eFrom an IRC conversation today with Moritz regarding the use of firejail for this purpose:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003e11:30 <moritzm> if not sure if it's the most viable option, the focus of firejail is on isolation from a security perspective. but I doubt its  possibilities to configure resource distribution\/consumption are sufficient for your use case\u003cbr \/\u003e\n11:30 <moritzm> e.g. what happens if one of the compartmented nodes is highly loaded\u003cbr \/\u003e\n11:31 <moritzm> to ensure that it doesn't harm the others\u003cbr \/\u003e\n11:32 <moritzm> firejail is designed to  ensure that if a firejailed process is compromised, it's unable to attack other bits of the system\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eAFAIK, not even container-based solutions can fully help in this edge case.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWe also don't need absolute isolation at any cost. Right now we have *zero* isolation as we are only running a single process. Even simple processes without any container magic offer a lot better isolation than that.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003e11:33 <moritzm> also, you'll probably need to run all nodes on the same port for Cassandra?\u003cbr \/\u003e\n11:33 <urandom> they'll each need their own IP, yeah\u003cbr \/\u003e\n11:33 <moritzm> going with systemd nspawn containers or a full-blown virtualisation seems more appropriate to me\u003c\/p\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe don't strictly need separate system images, which is why firejail is attractive as an intermediate step. It offers network namespaces, which is the main thing we need for Cassandra. We could also use docker or systemd-nspawn, but then need to deal with image distribution, for example by running our own docker registry. With firejail, we could defer tackling that.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_333\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1467127\" id=\"1467127\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e mentioned this in \u003ca href=\"\/T102306\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003eT102306: Services team roadmap July - September 2015 (Q1 2015\/16)\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1467127\" data-sigil=\"has-tooltip\" data-meta=\"0_331\"\u003e\u003cspan class=\"screen-only\"\u003eJul 21 2015, 4:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-07-21 16:00:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_342\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1524978\" id=\"1524978\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1524978\" data-sigil=\"has-tooltip\" data-meta=\"0_340\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2015, 6:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-10 18:55:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_338\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe talked about this at the last hardware planning meeting. The consensus was to keep things simple for now, and set up additional IPs & bind Cassandra instances to those. The exact startup solution remains tbd, but one option might be to leverage \u003ca href=\"http:\/\/0pointer.de\/blog\/projects\/instances.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/0pointer.de\/blog\/projects\/instances.html\u003c\/a\u003e. We should be able to test this in labs. If we get this ready soon, then we could consider leveraging this to test cross-DC replication with fewer temporary nodes in codfw.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/mobrovac\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@mobrovac\u003c\/span\u003e\u003c\/a\u003e: any thoughts \/ interest?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_345\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1525011\" id=\"1525011\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-up phui-timeline-icon\" data-meta=\"0_344\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e raised the priority of this task from \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1525011\" data-sigil=\"has-tooltip\" data-meta=\"0_343\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2015, 7:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-10 19:00:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_348\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1525099\" id=\"1525099\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_347\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e mentioned this in \u003ca href=\"\/T108613\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003eT108613: Set up multi-DC replication for Cassandra\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1525099\" data-sigil=\"has-tooltip\" data-meta=\"0_346\"\u003e\u003cspan class=\"screen-only\"\u003eAug 10 2015, 7:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-10 19:34:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_357\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1527183\" id=\"1527183\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_99\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1527183\" data-sigil=\"has-tooltip\" data-meta=\"0_355\"\u003e\u003cspan class=\"screen-only\"\u003eAug 11 2015, 12:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-11 12:43:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_353\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_354\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_100\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1524978\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1524978\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eWe talked about this at the last hardware planning meeting. The consensus was to keep things simple for now, and set up additional IPs & bind Cassandra instances to those. The exact startup solution remains tbd, but one option might be to leverage \u003ca href=\"http:\/\/0pointer.de\/blog\/projects\/instances.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/0pointer.de\/blog\/projects\/instances.html\u003c\/a\u003e. We should be able to test this in labs. If we get this ready soon, then we could consider leveraging this to test cross-DC replication with fewer temporary nodes in codfw.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Joe\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Joe\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/mobrovac\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@mobrovac\u003c\/span\u003e\u003c\/a\u003e: any thoughts \/ interest?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eyeah I'll look into this, using system units and possibly native systemd instances, another approach is what we are using for varnish (i.e. the instance name is part of the service name)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_366\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1527327\" id=\"1527327\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1527327\" data-sigil=\"has-tooltip\" data-meta=\"0_364\"\u003e\u003cspan class=\"screen-only\"\u003eAug 11 2015, 2:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-11 14:10:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_362\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_363\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_102\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1524978\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1524978\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThe exact startup solution remains tbd, but one option might be to leverage \u003ca href=\"http:\/\/0pointer.de\/blog\/projects\/instances.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/0pointer.de\/blog\/projects\/instances.html\u003c\/a\u003e.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI was thinking the same. It's the most straightforward solution at this point. Leveraging SystemD instances would allow us to easily select different config files for each instance. In order to get there we'd need to:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eassign multiple IPs to each host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eswitch to SystemD for Cassandra (obviously)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003emodify Cassandra's Puppet module to be able to deploy multiple config files and set up the SystemD instances\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThe last bullet will take some doing, but all in all it's quite doable.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIf we get this ready soon, then we could consider leveraging this to test cross-DC replication with fewer temporary nodes in codfw.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eYup. In theory, once the codfw hardware is in, we are going to be able to production-test the multiple-instances-per-node idea.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_369\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/lvt5pyizsk3uckl4msuc\/PHID-FILE-eiyvx2uovr7zblyb6rkl\/profile-Kosiaris%2C_Alexandros_Sept_2013.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/akosiaris\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1527572\" id=\"1527572\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-times phui-timeline-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/akosiaris\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_103\"\u003eakosiaris\u003c\/a\u003e unsubscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1527572\" data-sigil=\"has-tooltip\" data-meta=\"0_367\"\u003e\u003cspan class=\"screen-only\"\u003eAug 11 2015, 3:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-11 15:16:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_379\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1532040\" id=\"1532040\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_378\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1532040\" data-sigil=\"has-tooltip\" data-meta=\"0_377\"\u003e\u003cspan class=\"screen-only\"\u003eAug 12 2015, 4:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-12 16:50:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_375\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_376\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI'm trying the systemd instances first, (no puppet code review yet), basically the idea is to set \u003ctt class=\"remarkup-monospaced\"\u003eCASSANDRA_CONF\u003c\/tt\u003e to point to a separate config directory. The \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/cassandra.in.sh\u003c\/tt\u003e file is the same as \u003ctt class=\"remarkup-monospaced\"\u003e\/usr\/share\/cassandra\/cassandra.in.sh\u003c\/tt\u003e (the default) except that it won't assign \u003ctt class=\"remarkup-monospaced\"\u003eCASSANDRA_CONF\u003c\/tt\u003e.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ cat \/etc\/systemd\/system\/cassandra@.service \u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[Unit]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eDescription=distributed storage system for structured data\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[Service]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUser=cassandra\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eExecStart=\/usr\/sbin\/cassandra -f\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eEnvironment="CASSANDRA_INCLUDE=\/etc\/cassandra.in.sh"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eEnvironment="CASSANDRA_CONF=\/etc\/cassandra.%i"\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ethis works, though we will have to change the jmx port per-instance, currently the jvm doesn't allow to bind jmx to a particular address\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ grep -ir -e \/var\/lib -e ' 127' -e ^JMX_PORT \/etc\/cassandra.a\/*\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra-env.sh:JMX_PORT="7199"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra.yaml:    - \/var\/lib\/cassandra.a\/data\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra.yaml:commitlog_directory: \/var\/lib\/cassandra.a\/commitlog\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra.yaml:saved_caches_directory: \/var\/lib\/cassandra.a\/saved_caches\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra.yaml:         - seeds: 127.0.0.10,127.0.0.11,127.0.0.12\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra.yaml:listen_address: 127.0.0.10\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\/etc\/cassandra.a\/cassandra.yaml:rpc_address: 127.0.0.10\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003efor \u003ctt class=\"remarkup-monospaced\"\u003enodetool\u003c\/tt\u003e and other admin utils we should be able to leverage the same \u003ctt class=\"remarkup-monospaced\"\u003eCASSANDRA_INCLUDE\u003c\/tt\u003e variable by creating a wrapper to set that accordingly, in other words something like this\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ cat nodetool.\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e#!\/bin\/sh\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eset -e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eset -u\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eme=$(basename $0)\u003c\/span\u003e\n\u003cspan class=\"go\"\u003einstance=${me#*.}\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eexport CASSANDRA_INCLUDE=\/etc\/cassandra.in.sh\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eexport CASSANDRA_CONF=\/etc\/cassandra.${instance}\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eexec nodetool "$@"\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_388\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1535221\" id=\"1535221\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_387\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1535221\" data-sigil=\"has-tooltip\" data-meta=\"0_386\"\u003e\u003cspan class=\"screen-only\"\u003eAug 13 2015, 2:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-13 14:59:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_384\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_385\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ealso multiple instances means we'll need to adapt \u003ctt class=\"remarkup-monospaced\"\u003ecassandra-metrics-collector\u003c\/tt\u003e to discover and work with those, I think \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e might speak to that too\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_397\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1536539\" id=\"1536539\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_396\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1536539\" data-sigil=\"has-tooltip\" data-meta=\"0_395\"\u003e\u003cspan class=\"screen-only\"\u003eAug 13 2015, 7:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-13 19:04:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_393\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_394\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_109\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1535221\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1535221\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003ealso multiple instances means we'll need to adapt \u003ctt class=\"remarkup-monospaced\"\u003ecassandra-metrics-collector\u003c\/tt\u003e to discover and work with those, I think \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e might speak to that too\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eRight, so we could either install multiple cron jobs, each with an invocation that matches an instance of Cassandra (with corresponding JMX port and prefix), which seems kind of janky, or we could discover the locally running instances of Cassandra, and automatically collect from each concurrently.  The latter could be done with a wrapper script and jps(1) for discovery, or from Java and incorporated directly into \u003ca href=\"https:\/\/github.com\/wikimedia\/cassandra-metrics-collector\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecassandra-metrics-collector\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cp\u003eThe discovery approach seems much cleaner to me.  We would need to puppetize an additional flag to Cassandra startup (say \u003ctt class=\"remarkup-monospaced\"\u003e-Dinstance.name\u003c\/tt\u003e to use as a prefix), but that might be a good idea, anyway.\u003c\/p\u003e\n\n\u003cp\u003eIncorporating this into \u003ca href=\"https:\/\/github.com\/wikimedia\/cassandra-metrics-collector\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecassandra-metrics-collector\u003c\/a\u003e might make for a good opportunity to clean things up there, (what we have now amounts to a MVP, and was put in place to get us out of a jam).  For example, it'd be cleaner still to make it run as a daemon and schedule it's collection(s), as opposed to being run from cron.\u003c\/p\u003e\n\n\u003cp\u003eEither way, I don't think we're talking about more than a few hours work.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_406\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1538970\" id=\"1538970\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_405\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003egerritbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1538970\" data-sigil=\"has-tooltip\" data-meta=\"0_404\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2015, 10:18 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-14 10:18:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_402\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_403\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_110\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 231512 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncassandra: WIP support for multiple instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/231512\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/231512\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_409\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1538971\" id=\"1538971\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_408\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_113\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1538971\" data-sigil=\"has-tooltip\" data-meta=\"0_407\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2015, 10:18 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-14 10:18:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_418\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1539350\" id=\"1539350\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_417\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1539350\" data-sigil=\"has-tooltip\" data-meta=\"0_416\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2015, 12:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-14 12:23:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_414\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_415\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1532040\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1532040\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI'm trying the systemd instances first, (no puppet code review yet), basically the idea is to set \u003ctt class=\"remarkup-monospaced\"\u003eCASSANDRA_CONF\u003c\/tt\u003e to point to a separate config directory. The \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/cassandra.in.sh\u003c\/tt\u003e file is the same as \u003ctt class=\"remarkup-monospaced\"\u003e\/usr\/share\/cassandra\/cassandra.in.sh\u003c\/tt\u003e (the default) except that it won't assign \u003ctt class=\"remarkup-monospaced\"\u003eCASSANDRA_CONF\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/231512\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/231512\u003c\/a\u003e contains a WIP version of this, however I'm not yet sure how we should approach the puppet part wrt multiple instances, thoughts?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_427\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1539380\" id=\"1539380\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_426\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1539380\" data-sigil=\"has-tooltip\" data-meta=\"0_425\"\u003e\u003cspan class=\"screen-only\"\u003eAug 14 2015, 12:34 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-14 12:34:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_423\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_424\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1539350\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1539350\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI'm not yet sure how we should approach the puppet part wrt multiple instances, thoughts?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWhat springs to mind is:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003emake a \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e define, which would basically be the modified version of the current \u003ctt class=\"remarkup-monospaced\"\u003ecassandra\u003c\/tt\u003e class\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ein the \u003ctt class=\"remarkup-monospaced\"\u003ecassandra\u003c\/tt\u003e class, simply use one \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e resource, so that it's backwards-compatible (and if others want to use Cassandra in a \u003cem\u003enormal\u003c\/em\u003e set-up)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003emake a second \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::restbase\u003c\/tt\u003e class (or, better yet, \u003ctt class=\"remarkup-monospaced\"\u003erestbase::cassandra\u003c\/tt\u003e) which would set up all of the instances on one node (we'd need to know all of the instances' IPs \u003cem\u003ea priory\u003c\/em\u003e, and could just pass it in to \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThat would imply that we'd need X \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e resource blocks in \u003ctt class=\"remarkup-monospaced\"\u003erestbase::cassandra\u003c\/tt\u003e for X instances, but it could be a good start.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_436\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1546025\" id=\"1546025\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1546025\" data-sigil=\"has-tooltip\" data-meta=\"0_434\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2015, 4:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-17 16:21:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_432\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_119\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1539380\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1539380\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/mobrovac\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@mobrovac\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1539350\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1539350\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI'm not yet sure how we should approach the puppet part wrt multiple instances, thoughts?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWhat springs to mind is:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003emake a \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e define, which would basically be the modified version of the current \u003ctt class=\"remarkup-monospaced\"\u003ecassandra\u003c\/tt\u003e class\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ein the \u003ctt class=\"remarkup-monospaced\"\u003ecassandra\u003c\/tt\u003e class, simply use one \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e resource, so that it's backwards-compatible (and if others want to use Cassandra in a \u003cem\u003enormal\u003c\/em\u003e set-up)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003emake a second \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::restbase\u003c\/tt\u003e class (or, better yet, \u003ctt class=\"remarkup-monospaced\"\u003erestbase::cassandra\u003c\/tt\u003e) which would set up all of the instances on one node (we'd need to know all of the instances' IPs \u003cem\u003ea priory\u003c\/em\u003e, and could just pass it in to \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003ethat could work, but how would hiera look like in this case? there's only one define for \u003ctt class=\"remarkup-monospaced\"\u003ecassandra::instance\u003c\/tt\u003e so I'm not sure how we could pass different parameters for different instances\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_447\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_445\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_446\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1546040\" id=\"1546040\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_120\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1546040\" data-sigil=\"has-tooltip\" data-meta=\"0_443\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2015, 4:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-17 16:24:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_441\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e hiera autolookup obviously doesn't work for defines, only for classes - so you'd need to define explicitly all the instances on one server; the trick is to make it easy to define such parameters.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_456\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1546867\" id=\"1546867\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1546867\" data-sigil=\"has-tooltip\" data-meta=\"0_454\"\u003e\u003cspan class=\"screen-only\"\u003eAug 17 2015, 6:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-17 18:56:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_452\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt would probably be sufficient to have one seed instance per physical node only, and let the client discover the other instances dynamically. Maybe we could model this as one primary plus some replicas? That way the 'single instance' case could perhaps be handled by the same class, without 'extra instances' defined.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_459\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1548359\" id=\"1548359\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003efgiunchedi\u003c\/a\u003e added a parent task: \u003ca href=\"\/T108306\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003eT108306: better cassandra process checks\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1548359\" data-sigil=\"has-tooltip\" data-meta=\"0_457\"\u003e\u003cspan class=\"screen-only\"\u003eAug 18 2015, 9:34 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-18 09:34:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_468\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1557475\" id=\"1557475\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_467\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1557475\" data-sigil=\"has-tooltip\" data-meta=\"0_466\"\u003e\u003cspan class=\"screen-only\"\u003eAug 20 2015, 4:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-20 16:04:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_464\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_465\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_127\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1536539\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1536539\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThe discovery approach seems much cleaner to me.  We would need to puppetize an additional flag to Cassandra startup (say \u003ctt class=\"remarkup-monospaced\"\u003e-Dinstance.name\u003c\/tt\u003e to use as a prefix), but that might be a good idea, anyway.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI agree we should be discovering jvms running via the java api, fetch instance name property via jmx and collect metrics as usual\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIncorporating this into \u003ca href=\"https:\/\/github.com\/wikimedia\/cassandra-metrics-collector\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecassandra-metrics-collector\u003c\/a\u003e might make for a good opportunity to clean things up there, (what we have now amounts to a MVP, and was put in place to get us out of a jam).  For example, it'd be cleaner still to make it run as a daemon and schedule it's collection(s), as opposed to being run from cron.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003esince we're potentially talking 4\/5 instances I think having a daemon makes sense now over cron\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_477\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1557554\" id=\"1557554\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_476\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_128\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1557554\" data-sigil=\"has-tooltip\" data-meta=\"0_475\"\u003e\u003cspan class=\"screen-only\"\u003eAug 20 2015, 4:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-20 16:17:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_473\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_474\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_129\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eRegarding metrics collection: Do we actually still need a separate collector? The regular collector worked fine in staging, and I think there is a decent chance that with current moderate instance sizes it will work in production as well.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_486\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1558891\" id=\"1558891\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1558891\" data-sigil=\"has-tooltip\" data-meta=\"0_484\"\u003e\u003cspan class=\"screen-only\"\u003eAug 20 2015, 9:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-20 21:46:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_482\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_483\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1557554\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1557554\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eRegarding metrics collection: Do we actually still need a separate collector? The regular collector worked fine in staging, and I think there is a decent chance that with current moderate instance sizes it will work in production as well.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYeah, that's a reasonable question (and one I've asked myself).  I'd rather not maintain something separate, but on the other hand, we've made a number of fixes\/enhancements in \u003ca href=\"https:\/\/github.com\/wikimedia\/cassandra-metrics-collector\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecassandra-metrics-collector\u003c\/a\u003e that we'd have to give up.  For example, \u003ca href=\"\/T97024\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT97024\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e introduced working histograms for column count and row size (\u003ca href=\"http:\/\/grafana.wikimedia.org\/#\/dashboard\/db\/restbase-cassandra-cf-rowcolumn-size\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/grafana.wikimedia.org\/#\/dashboard\/db\/restbase-cassandra-cf-rowcolumn-size\u003c\/a\u003e), and the \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T101764#1416708\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebroken non-heap usage\u003c\/a\u003e was the result of a bug in the way dropwizard calculates that value, (manifesting after the switch to JDK 8).\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/issues.apache.org\/jira\/browse\/CASSANDRA-9625\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eCASSANDRA-9625\u003c\/a\u003e has also accumulated a number of "me toos" since I reported the issue, and doesn't appear to have moved any closer to the finish line.  This doesn't come as a complete surprise, the defacto method of collecting metrics in Cassandra is JMX, every other mechanism is less tested and likely to be supported on a best-effort basis.\u003c\/p\u003e\n\n\u003chr class=\"remarkup-hr\" \/\u003e\n\n\u003cp\u003eFWIW: I just put up a prototype of discovery at \u003ca href=\"https:\/\/github.com\/eevans\/cassandra-metrics-collector\/tree\/discovery\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/eevans\/cassandra-metrics-collector\/tree\/discovery\u003c\/a\u003e. It uses the \u003ca href=\"http:\/\/docs.oracle.com\/javase\/8\/docs\/jdk\/api\/attach\/spec\/index.html?com\/sun\/tools\/attach\/package-summary.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eattach api\u003c\/a\u003e to obtain and introspect all of the JVMs running on a system.  For those that have a main class of \u003ctt class=\"remarkup-monospaced\"\u003eorg.apache.cassandra.service.CassandraDaemon\u003c\/tt\u003e, (and a \u003ctt class=\"remarkup-monospaced\"\u003e-Dcassandra.instance-name\u003c\/tt\u003e property), it collects metrics and pushes them to Graphite.  And, it does this all over domain sockets (which has me thinking of a \u003ctt class=\"remarkup-monospaced\"\u003enodetool\u003c\/tt\u003e enhancement to do the same, and in so doing, solve our security concerns with JMX).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_495\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1575788\" id=\"1575788\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_494\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1575788\" data-sigil=\"has-tooltip\" data-meta=\"0_493\"\u003e\u003cspan class=\"screen-only\"\u003eAug 26 2015, 3:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-26 15:40:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_491\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_492\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ethe way it works now in \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/#\/c\/231512\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/#\/c\/231512\/\u003c\/a\u003e is that an instance named "default" will behave exactly like stock cassandra, otherwise several separate instances are created:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003efilippo@test-cassandra3:~$ systemctl status cassandra-a\n\u25cf cassandra-a.service - distributed storage system for structured data\n   Loaded: loaded (\/etc\/systemd\/system\/cassandra-a.service; disabled)\n   Active: active (running) since Wed 2015-08-26 08:06:25 UTC; 7h ago\n Main PID: 15787 (java)\n   CGroup: \/system.slice\/cassandra-a.service\n           \u2514\u250015787 java -ea -javaagent:\/usr\/share\/cassandra\/lib\/jamm-0.3.0.jar -XX:+CMSClassUnloadingEnabled -XX:+UseThreadPrior...\nfilippo@test-cassandra3:~$ systemctl status cassandra-b\n\u25cf cassandra-b.service - distributed storage system for structured data\n   Loaded: loaded (\/etc\/systemd\/system\/cassandra-b.service; disabled)\n   Active: active (running) since Wed 2015-08-26 08:06:30 UTC; 7h ago\n Main PID: 15910 (java)\n   CGroup: \/system.slice\/cassandra-b.service\n           \u2514\u250015910 java -ea -javaagent:\/usr\/share\/cassandra\/lib\/jamm-0.3.0.jar -XX:+CMSClassUnloadingEnabled -XX:+UseThreadPrior...\nfilippo@test-cassandra3:~$ ls -la \/var\/lib\/cassandra-a\nls: cannot open directory \/var\/lib\/cassandra-a: Permission denied\nfilippo@test-cassandra3:~$ sudo ls -la \/var\/lib\/cassandra-a\ntotal 20\ndrwxr-x---  5 cassandra cassandra 4096 Aug 25 16:28 .\ndrwxr-xr-x 42 root      root      4096 Aug 25 17:01 ..\ndrwxr-x---  2 cassandra cassandra 4096 Aug 26 08:06 commitlog\ndrwxr-x---  5 cassandra cassandra 4096 Aug 25 16:29 data\ndrwxr-x---  2 cassandra cassandra 4096 Aug 26 12:06 saved_caches\nfilippo@test-cassandra3:~$ sudo ls -la \/var\/lib\/cassandra-b\ntotal 20\ndrwxr-x---  5 cassandra cassandra 4096 Aug 25 17:01 .\ndrwxr-xr-x 42 root      root      4096 Aug 25 17:01 ..\ndrwxr-x---  2 cassandra cassandra 4096 Aug 26 08:06 commitlog\ndrwxr-x---  5 cassandra cassandra 4096 Aug 25 17:01 data\ndrwxr-x---  2 cassandra cassandra 4096 Aug 26 12:06 saved_caches\nfilippo@test-cassandra3:~$\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_504\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1580153\" id=\"1580153\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_503\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1580153\" data-sigil=\"has-tooltip\" data-meta=\"0_502\"\u003e\u003cspan class=\"screen-only\"\u003eAug 27 2015, 3:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-27 15:07:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_500\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_501\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 234286 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\nWIP: xenon additional instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/234286\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/234286\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_513\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1580211\" id=\"1580211\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_512\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1580211\" data-sigil=\"has-tooltip\" data-meta=\"0_511\"\u003e\u003cspan class=\"screen-only\"\u003eAug 27 2015, 3:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-08-27 15:24:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_509\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_510\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 234292 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\nWIP: xenon additional instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/234292\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/234292\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_522\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1594988\" id=\"1594988\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_521\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1594988\" data-sigil=\"has-tooltip\" data-meta=\"0_520\"\u003e\u003cspan class=\"screen-only\"\u003eSep 1 2015, 10:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-09-01 22:05:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_518\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_519\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1536539\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1536539\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1535221\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1535221\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003ealso multiple instances means we'll need to adapt \u003ctt class=\"remarkup-monospaced\"\u003ecassandra-metrics-collector\u003c\/tt\u003e to discover and work with those, I think \u003ca href=\"\/p\/Eevans\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Eevans\u003c\/span\u003e\u003c\/a\u003e might speak to that too\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eRight, so we could either install multiple cron jobs, each with an invocation that matches an instance of Cassandra (with corresponding JMX port and prefix), which seems kind of janky, or we could discover the locally running instances of Cassandra, and automatically collect from each concurrently.  The latter could be done with a wrapper script and jps(1) for discovery, or from Java and incorporated directly into \u003ca href=\"https:\/\/github.com\/wikimedia\/cassandra-metrics-collector\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecassandra-metrics-collector\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cp\u003eThe discovery approach seems much cleaner to me.  We would need to puppetize an additional flag to Cassandra startup (say \u003ctt class=\"remarkup-monospaced\"\u003e-Dinstance.name\u003c\/tt\u003e to use as a prefix), but that might be a good idea, anyway.\u003c\/p\u003e\n\n\u003cp\u003eIncorporating this into \u003ca href=\"https:\/\/github.com\/wikimedia\/cassandra-metrics-collector\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecassandra-metrics-collector\u003c\/a\u003e might make for a good opportunity to clean things up there, (what we have now amounts to a MVP, and was put in place to get us out of a jam).  For example, it'd be cleaner still to make it run as a daemon and schedule it's collection(s), as opposed to being run from cron.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eMy latest is here: \u003ca href=\"https:\/\/github.com\/eevans\/cassandra-metrics-collector\/tree\/discovery\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/eevans\/cassandra-metrics-collector\/tree\/discovery\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eTo use the daemon, each instance of Cassandra must be parameterized to supply a \u003ctt class=\"remarkup-monospaced\"\u003e-Dcassandra.instance-id=<name>\u003c\/tt\u003e property, which is used to provide stable naming for the Graphite prefix.  All locally running instances of Cassandra are discovered on startup.  Metrics are collected at \u003ctt class=\"remarkup-monospaced\"\u003e{ -i, --interval }\u003c\/tt\u003e seconds and written to the Carbon server specified by \u003ctt class=\"remarkup-monospaced\"\u003e{ -H, --carbon-host }\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003e{ -p, --carbon-port }\u003c\/tt\u003e.  Internal statistics (successes and failures) are likewise written to Carbon at \u003ctt class=\"remarkup-monospaced\"\u003e{ -i, --interval }\u003c\/tt\u003e seconds in the form of \u003ctt class=\"remarkup-monospaced\"\u003ecmcd.instances.<name>.{success,failure}\u003c\/tt\u003e.  Periodic rediscovery (required to pick up new instances, or restarts of existing instances) occurs every \u003ctt class=\"remarkup-monospaced\"\u003e{ -di, --discovery-interval }\u003c\/tt\u003e seconds.\u003c\/p\u003e\n\n\u003cp\u003eWhile it's still possible to cron this against a single instance using the TCP connection information, the main-class has changed, and so it would require a minor change to the existing wrapper to get backward compatible behavior.  For this reason I've bumped the version to 2, and I've held off on opening a pull request (and subsequently a merge), until we have a plan for rollout.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_531\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1684713\" id=\"1684713\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_530\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1684713\" data-sigil=\"has-tooltip\" data-meta=\"0_529\"\u003e\u003cspan class=\"screen-only\"\u003eSep 29 2015, 12:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-09-29 12:55:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_527\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_528\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 242117 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\nrestbase-test2001 additional cassandra instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/242117\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/242117\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_534\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1684759\" id=\"1684759\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_533\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_142\"\u003efgiunchedi\u003c\/a\u003e closed subtask \u003ca href=\"\/T113939\" class=\"phui-handle handle-status-closed\"\u003eT113939: assess impact of many cassandra seed nodes with multi instance\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1684759\" data-sigil=\"has-tooltip\" data-meta=\"0_532\"\u003e\u003cspan class=\"screen-only\"\u003eSep 29 2015, 1:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-09-29 13:09:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_543\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1689431\" id=\"1689431\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_542\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_143\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1689431\" data-sigil=\"has-tooltip\" data-meta=\"0_541\"\u003e\u003cspan class=\"screen-only\"\u003eSep 30 2015, 2:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-09-30 14:49:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_539\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_540\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_144\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 242117 merged by Filippo Giunchedi:\u003cbr \/\u003e\nrestbase-test2001 additional cassandra instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/242117\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/242117\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_546\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1689442\" id=\"1689442\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_545\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_145\"\u003efgiunchedi\u003c\/a\u003e mentioned this in \u003ca href=\"\/rODNS8ed317d5ced72dac2dfaa84d3c068bd0f8c62d63\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_146\"\u003erODNS8ed317d5ced7: restbase-test2001 additional cassandra instances\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1689442\" data-sigil=\"has-tooltip\" data-meta=\"0_544\"\u003e\u003cspan class=\"screen-only\"\u003eSep 30 2015, 2:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-09-30 14:53:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_556\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1691567\" id=\"1691567\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_555\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_147\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1691567\" data-sigil=\"has-tooltip\" data-meta=\"0_554\"\u003e\u003cspan class=\"screen-only\"\u003eSep 30 2015, 11:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-09-30 23:16:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_552\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_553\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_148\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAs an aside, \u003ca href=\"http:\/\/www.scylladb.com\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/www.scylladb.com\/\u003c\/a\u003e are working on a C++ Cassandra clone with a one-instance-per-core architecture. Part of their motivation for the instance-per-core design despite having no GC to worry about is avoiding remote NUMA memory accesses and lock contention. Their \u003ca href=\"http:\/\/www.scylladb.com\/technology\/cassandra-vs-scylla-benchmark\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebenchmarks\u003c\/a\u003e are looking good, but some of the delta seems to be coming from them comparing to a single Cassandra instance on relatively beefy hardware. I wouldn't be surprised if a multi-instance Cassandra setup came within a more reasonable factor (2-3x) of their implementation.\u003c\/p\u003e\n\n\u003cp\u003eIn any case, looking forward to scylladb maturing & catching up on features we need.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_565\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1693365\" id=\"1693365\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_149\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1693365\" data-sigil=\"has-tooltip\" data-meta=\"0_563\"\u003e\u003cspan class=\"screen-only\"\u003eOct 1 2015, 2:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-01 14:40:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_561\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_562\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 242896 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncassandra: add multi-instance support, disabled\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/242896\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/242896\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_574\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1701518\" id=\"1701518\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_151\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1701518\" data-sigil=\"has-tooltip\" data-meta=\"0_572\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2015, 9:15 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-05 09:15:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_570\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_571\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_152\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 242896 merged by Filippo Giunchedi:\u003cbr \/\u003e\ncassandra: add multi-instance support, disabled\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/242896\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/242896\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_577\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1701519\" id=\"1701519\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_576\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_153\"\u003efgiunchedi\u003c\/a\u003e mentioned this in \u003ca href=\"\/rOPUP29bd45f080b32019a116e819ef5559745b772d93\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003erOPUP29bd45f080b3: cassandra: add multi-instance support, disabled\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1701519\" data-sigil=\"has-tooltip\" data-meta=\"0_575\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2015, 9:15 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-05 09:15:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_586\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1702183\" id=\"1702183\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_585\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_155\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1702183\" data-sigil=\"has-tooltip\" data-meta=\"0_584\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2015, 3:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-05 15:27:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_582\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_583\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 243675 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncassandra: enable multi-instance\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/243675\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/243675\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_595\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1705697\" id=\"1705697\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_594\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_157\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1705697\" data-sigil=\"has-tooltip\" data-meta=\"0_593\"\u003e\u003cspan class=\"screen-only\"\u003eOct 6 2015, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-06 15:58:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_591\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_592\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_158\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 243944 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncassandra: add restbase-test2001 instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/243944\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/243944\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_598\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/u5w4jepkaw7v43o6a5sg\/PHID-FILE-5qligaxnzh27mxijmb2j\/alphanumeric_aleo-white_J.png-_299a89-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jayvdb\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1707815\" id=\"1707815\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_597\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jayvdb\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_159\"\u003ejayvdb\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1707815\" data-sigil=\"has-tooltip\" data-meta=\"0_596\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2015, 4:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-07 04:58:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_607\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1708242\" id=\"1708242\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_606\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_160\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1708242\" data-sigil=\"has-tooltip\" data-meta=\"0_605\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2015, 10:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-07 10:12:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_603\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_604\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 243675 merged by Filippo Giunchedi:\u003cbr \/\u003e\ncassandra: enable multi-instance\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/243675\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/243675\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_610\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1708244\" id=\"1708244\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_609\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_162\"\u003efgiunchedi\u003c\/a\u003e mentioned this in \u003ca href=\"\/rOPUPc03e7731628f87b352b0a4fe849cf6ea8c88ea57\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_163\"\u003erOPUPc03e7731628f: cassandra: enable multi-instance\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1708244\" data-sigil=\"has-tooltip\" data-meta=\"0_608\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2015, 10:13 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-07 10:13:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_619\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1708628\" id=\"1708628\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_618\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_164\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1708628\" data-sigil=\"has-tooltip\" data-meta=\"0_617\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2015, 1:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-07 13:29:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_615\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_616\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_165\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 243944 merged by Filippo Giunchedi:\u003cbr \/\u003e\ncassandra: add restbase-test2001 instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/243944\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/243944\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_622\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1708629\" id=\"1708629\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_621\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_166\"\u003efgiunchedi\u003c\/a\u003e mentioned this in \u003ca href=\"\/rOPUPe7b17733fe4529affc0098a94915b2089985c09c\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_167\"\u003erOPUPe7b17733fe45: cassandra: add restbase-test2001 instances\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1708629\" data-sigil=\"has-tooltip\" data-meta=\"0_620\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2015, 1:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-07 13:30:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_631\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1711913\" id=\"1711913\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_630\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_168\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1711913\" data-sigil=\"has-tooltip\" data-meta=\"0_629\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2015, 9:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-08 09:48:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_627\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_628\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 244420 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncodfw: add test cassandra instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/244420\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/244420\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_640\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1711924\" id=\"1711924\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_639\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_170\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1711924\" data-sigil=\"has-tooltip\" data-meta=\"0_638\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2015, 9:50 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-08 09:50:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_636\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_637\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 244420 merged by Filippo Giunchedi:\u003cbr \/\u003e\ncodfw: add test cassandra instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/244420\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/244420\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_643\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1713076\" id=\"1713076\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_642\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_172\"\u003eEevans\u003c\/a\u003e mentioned this in \u003ca href=\"\/T93886\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_173\"\u003eT93886: Cassandra\/CQL query interface monitoring\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1713076\" data-sigil=\"has-tooltip\" data-meta=\"0_641\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2015, 5:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-08 17:51:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_652\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1714689\" id=\"1714689\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_651\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_174\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1714689\" data-sigil=\"has-tooltip\" data-meta=\"0_650\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2015, 8:35 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-09 08:35:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_648\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_649\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_175\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 231512 abandoned by Filippo Giunchedi:\u003cbr \/\u003e\ncassandra: WIP support for multiple instances\u003c\/p\u003e\n\n\u003cp\u003eReason:\u003cbr \/\u003e\npartially merged already\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/231512\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/231512\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_661\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1715423\" id=\"1715423\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_660\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_176\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1715423\" data-sigil=\"has-tooltip\" data-meta=\"0_659\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2015, 3:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-09 15:32:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_657\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_658\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003erestbase-test2001.codfw.wmnet has now been configured for 2 instances ({a,b}), but something is amiss with the data sizes.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eeevans@restbase-test2001:~$ nodetool-a status local_group_wikipedia_T_parsoid_html\nDatacenter: codfw\n=================\nStatus=Up\/Down\n|\/ State=Normal\/Leaving\/Joining\/Moving\n--  Address        Load       Tokens  Owns (effective)  Host ID                               Rack\nUN  10.192.16.150  147.02 GB  256     76.9%             4afd4cc2-fe90-4868-bb99-3bf6496d5a57  rack1\nUN  10.192.16.151  149.15 GB  256     75.6%             ec1d70ad-aab6-4f68-af44-2bc31b500804  rack1\nUN  10.192.16.154  2.41 GB    256     76.1%             3d5e1cbf-495d-4d48-a712-e489c909ff8a  rack1\nUN  10.192.16.155  1.67 GB    256     71.3%             11661295-f159-4fb8-8830-38a4ebc150cf  rack1\nDatacenter: eqiad\n=================\nStatus=Up\/Down\n|\/ State=Normal\/Leaving\/Joining\/Moving\n--  Address        Load       Tokens  Owns (effective)  Host ID                               Rack\nUN  10.64.16.147   150.94 GB  256     100.0%            235934bc-33f0-4744-b374-f7c96c8edc95  rack1\nUN  10.64.16.149   150.42 GB  256     100.0%            5be66290-4784-4fed-a4f1-d496ea09d803  rack1\nUN  10.64.0.200    149.7 GB   256     100.0%            20ec6bb4-195b-479c-a540-e6a7c70b589d  rack1\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cem\u003eNote: 10.192.16.15{4,5} is {a,b} respectively.\u003c\/em\u003e\u003cbr \/\u003e\n\u003cem\u003eNote: cleanups are underway for restbase-test200{2,3}, and load figures are expected to decrease there accordingly.\u003c\/em\u003e\u003c\/p\u003e\n\n\u003cp\u003eEffective ownership seems to be correct, but the load values for these new hosts are way off.\u003c\/p\u003e\n\n\u003cp\u003eWhile attempting to do a rebuild from 2001-a, the following exception(s) occurred.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eERROR [STREAM-IN-\/10.64.0.200] 2015-10-09 15:18:14,718 StreamSession.java:502 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Streaming error occurred\njava.io.IOException: CF 957dffd0-6d1d-11e5-afba-95b33dbe4433 was dropped during streaming\n        at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:71) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:48) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:38) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:250) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_66-internal]\nINFO  [MemtableFlushWriter:3] 2015-10-09 15:18:14,719 Memtable.java:382 - Completed flushing \/srv\/cassandra-a\/data\/system\/compactions_in_progress-55080ab05d9c388690a4acb25fe1f77b\/system-compactions_in_progress-t\nmp-ka-457-Data.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1444403738977, position=22008277)\nINFO  [STREAM-IN-\/10.64.0.200] 2015-10-09 15:18:14,719 StreamResultFuture.java:180 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Session with \/10.64.0.200 is complete\nWARN  [STREAM-IN-\/10.64.0.200] 2015-10-09 15:18:14,721 StreamResultFuture.java:207 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Stream failed\nERROR [RMI TCP Connection(6)-10.192.16.149] 2015-10-09 15:18:14,724 StorageService.java:1058 - Error while rebuilding node\norg.apache.cassandra.streaming.StreamException: Stream failed\n        at org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) ~[guava-16.0.jar:na]\n        at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) ~[guava-16.0.jar:na]\n        at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) ~[guava-16.0.jar:na]\n        at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) ~[guava-16.0.jar:na]\n        at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) ~[guava-16.0.jar:na]\n        at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:412) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.StreamSession.onError(StreamSession.java:507) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:268) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_66-internal]\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cem\u003eNote: subsequent attempts result in the same; this is reproducible.\u003c\/em\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_670\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1715480\" id=\"1715480\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_669\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_178\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1715480\" data-sigil=\"has-tooltip\" data-meta=\"0_668\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2015, 3:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-09 15:51:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_666\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_667\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAn additional point of data, in the logs on xenon.eqiad.wmnet (10.64.0.200), there are CF id mismatches relating to the CF in question (957dffd0-6d1d-11e5-afba-95b33dbe4433):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eINFO  [MigrationStage:1] 2015-10-07 18:02:33,664 DefsTables.java:373 - Loading org.apache.cassandra.config.CFMetaData@4d3c59ec[cfId=957dffd0-6d1d-11e5-afba-95b33dbe4433,ksName=local_group_wikisource_T_parsoid_st\nash_html,cfName=meta,cfType=Standard,comparator=org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType),org.apache.cassandra.db.ma\nrshal.UTF8Type),comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,mi\nnCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=_del, type=org.apache.cassandra.db.marshal.TimeUUIDType, kind=REGULAR, componentIndex=1, indexName=null, indexType=null}, Co\nlumnDefinition{name=value, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=1, indexName=null, indexType=null}, ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type\n, kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=tid, type=org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType), kind=CLUSTE\nRING_COLUMN, componentIndex=0, indexName=null, indexType=null}],compactionStrategyClass=class org.apache.cassandra.db.compaction.LeveledCompactionStrategy,compactionStrategyOptions={},compressionParameters={ssta\nble_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.1,memtableFlushPeriod=0,caching={"keys":"ALL", "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndex\nInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=false]\nINFO  [MigrationStage:1] 2015-10-07 18:02:33,665 ColumnFamilyStore.java:339 - Initializing local_group_wikisource_T_parsoid_stash_html.meta\nINFO  [CompactionExecutor:5051] 2015-10-07 18:02:33,674 CompactionTask.java:274 - Compacted 4 sstables to [\/var\/lib\/cassandra\/data\/system\/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697\/system-schema_colu\nmnfamilies-ka-129,].  67,140 bytes to 65,284 (~97% of original) in 24ms = 2.594153MB\/s.  85 total partitions merged to 83.  Partition merge counts were {1:81, 2:2, }\nINFO  [MigrationStage:1] 2015-10-07 18:02:33,692 ColumnFamilyStore.java:912 - Enqueuing flush of schema_keyspaces: 236 (0%) on-heap, 226 (0%) off-heap\nINFO  [MemtableFlushWriter:3462] 2015-10-07 18:02:33,692 Memtable.java:347 - Writing Memtable-schema_keyspaces@256756851(0.150KiB serialized bytes, 3 ops, 0%\/0% of on\/off-heap limit)\nINFO  [MemtableFlushWriter:3462] 2015-10-07 18:02:33,693 Memtable.java:382 - Completed flushing \/var\/lib\/cassandra\/data\/system\/schema_keyspaces-b0f2235744583cdb9631c43e59ce3676\/system-schema_keyspaces-tmp-ka-246\n-Data.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1444036710512, position=16912978)\nINFO  [MigrationStage:1] 2015-10-07 18:02:33,701 ColumnFamilyStore.java:912 - Enqueuing flush of schema_columnfamilies: 908 (0%) on-heap, 1694 (0%) off-heap\nINFO  [MemtableFlushWriter:3461] 2015-10-07 18:02:33,701 Memtable.java:347 - Writing Memtable-schema_columnfamilies@507190959(1.432KiB serialized bytes, 27 ops, 0%\/0% of on\/off-heap limit)\nINFO  [MemtableFlushWriter:3461] 2015-10-07 18:02:33,702 Memtable.java:382 - Completed flushing \/var\/lib\/cassandra\/data\/system\/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697\/system-schema_columnfamilies-\ntmp-ka-130-Data.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1444036710512, position=16912978)\nINFO  [CompactionExecutor:5049] 2015-10-07 18:02:33,708 CompactionTask.java:274 - Compacted 4 sstables to [\/var\/lib\/cassandra\/data\/system\/schema_columns-296e9c049bec3085827dc17d3df2122a\/system-schema_columns-ka-\n129,].  97,201 bytes to 96,667 (~99% of original) in 50ms = 1.843777MB\/s.  85 total partitions merged to 83.  Partition merge counts were {1:81, 2:2, }\nINFO  [MigrationStage:1] 2015-10-07 18:02:33,710 ColumnFamilyStore.java:912 - Enqueuing flush of schema_columns: 932 (0%) on-heap, 1471 (0%) off-heap\nINFO  [MemtableFlushWriter:3462] 2015-10-07 18:02:33,710 Memtable.java:347 - Writing Memtable-schema_columns@70557208(1.234KiB serialized bytes, 28 ops, 0%\/0% of on\/off-heap limit)\nINFO  [MemtableFlushWriter:3462] 2015-10-07 18:02:33,711 Memtable.java:382 - Completed flushing \/var\/lib\/cassandra\/data\/system\/schema_columns-296e9c049bec3085827dc17d3df2122a\/system-schema_columns-tmp-ka-130-Dat\na.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1444036710512, position=16913808)\nERROR [MigrationStage:1] 2015-10-07 18:02:33,730 CassandraDaemon.java:223 - Exception in thread Thread[MigrationStage:1,5,main]\njava.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\n        at org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1145) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.db.DefsTables.updateColumnFamily(DefsTables.java:422) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.db.DefsTables.mergeColumnFamilies(DefsTables.java:295) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.db.DefsTables.mergeSchemaInternal(DefsTables.java:194) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.db.DefsTables.mergeSchema(DefsTables.java:166) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.service.MigrationManager$2.runMayThrow(MigrationManager.java:394) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_66-internal]\n        at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_66-internal]\n        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[na:1.8.0_66-internal]\n        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_66-internal]\n        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_66-internal]\nCaused by: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\n        at org.apache.cassandra.config.CFMetaData.validateCompatility(CFMetaData.java:1233) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:1161) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1141) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        ... 11 common frames omitted\n\n[ ... ]\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_679\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1715483\" id=\"1715483\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1715483\" data-sigil=\"has-tooltip\" data-meta=\"0_677\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2015, 3:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-09 15:56:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_675\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_676\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eon the peer side (xenon in this case) this shows\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\" style=\" max-height: 12em; overflow: auto;\"\u003eINFO  [STREAM-IN-\/10.192.16.154] 2015-10-09 15:18:14,136 StreamResultFuture.java:166 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c ID#0] Prepare completed. Receiving 0 files(0 bytes), sending 957 files(119617545673 bytes)\nINFO  [STREAM-IN-\/10.192.16.154] 2015-10-09 15:18:14,753 StreamResultFuture.java:180 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Session with \/10.192.16.154 is complete\nERROR [STREAM-OUT-\/10.192.16.154] 2015-10-09 15:18:14,755 StreamSession.java:502 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Streaming error occurred\njava.net.SocketException: Connection reset\n        at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113) ~[na:1.8.0_66-internal]\n        at java.net.SocketOutputStream.write(SocketOutputStream.java:153) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.OutputRecord.write(OutputRecord.java:417) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:876) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:847) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:138) ~[na:1.8.0_66-internal]\n        at org.apache.cassandra.io.util.DataOutputStreamPlus.write(DataOutputStreamPlus.java:45) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.io.util.AbstractDataOutput.writeInt(AbstractDataOutput.java:211) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.io.compress.CompressionMetadata$ChunkSerializer.serialize(CompressionMetadata.java:471) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.io.compress.CompressionMetadata$ChunkSerializer.serialize(CompressionMetadata.java:466) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.compress.CompressionInfo$CompressionInfoSerializer.serialize(CompressionInfo.java:60) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.compress.CompressionInfo$CompressionInfoSerializer.serialize(CompressionInfo.java:47) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.FileMessageHeader$FileMessageHeaderSerializer.serialize(FileMessageHeader.java:135) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.FileMessageHeader$FileMessageHeaderSerializer.serialize(FileMessageHeader.java:120) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.OutgoingFileMessage.serialize(OutgoingFileMessage.java:88) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:49) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) [apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) [apache-cassandra-2.1.8.jar:2.1.8]\n        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_66-internal]\nWARN  [STREAM-IN-\/10.192.16.154] 2015-10-09 15:18:14,757 StreamResultFuture.java:207 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Stream failed\nERROR [STREAM-OUT-\/10.192.16.154] 2015-10-09 15:18:14,759 StreamSession.java:502 - [Stream #de8487e0-6e98-11e5-aebd-c5a9546d6a7c] Streaming error occurred\njavax.net.ssl.SSLException: Connection has been shutdown: javax.net.ssl.SSLException: java.net.SocketException: Connection reset\n        at sun.security.ssl.SSLSocketImpl.checkEOF(SSLSocketImpl.java:1541) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.checkWrite(SSLSocketImpl.java:1553) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:71) ~[na:1.8.0_66-internal]\n        at java.nio.channels.Channels$WritableByteChannelImpl.write(Channels.java:458) ~[na:1.8.0_66-internal]\n        at org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) [apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) [apache-cassandra-2.1.8.jar:2.1.8]\n        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_66-internal]\nCaused by: javax.net.ssl.SSLException: java.net.SocketException: Connection reset\n        at sun.security.ssl.Alerts.getSSLException(Alerts.java:208) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1906) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.handleException(SSLSocketImpl.java:1870) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.SSLSocketImpl.handleException(SSLSocketImpl.java:1815) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:128) ~[na:1.8.0_66-internal]\n        at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:138) ~[na:1.8.0_66-internal]\n        at org.apache.cassandra.io.util.DataOutputStreamPlus.write(DataOutputStreamPlus.java:45) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.io.util.AbstractDataOutput.writeInt(AbstractDataOutput.java:211) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.io.compress.CompressionMetadata$ChunkSerializer.serialize(CompressionMetadata.java:471) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.io.compress.CompressionMetadata$ChunkSerializer.serialize(CompressionMetadata.java:466) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.compress.CompressionInfo$CompressionInfoSerializer.serialize(CompressionInfo.java:60) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.compress.CompressionInfo$CompressionInfoSerializer.serialize(CompressionInfo.java:47) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.FileMessageHeader$FileMessageHeaderSerializer.serialize(FileMessageHeader.java:135) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.FileMessageHeader$FileMessageHeaderSerializer.serialize(FileMessageHeader.java:120) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.OutgoingFileMessage.serialize(OutgoingFileMessage.java:88) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:49) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) ~[apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) [apache-cassandra-2.1.8.jar:2.1.8]\n        at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) [apache-cassandra-2.1.8.jar:2.1.8]\n        ... 1 common frames omitted\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003emoreover the CF IDs involved show up in xenon's logs (\u003ctt class=\"remarkup-monospaced\"\u003e958a82f0-6d1d-11e5-b81f-81302e0f25c7\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003e957dffd0-6d1d-11e5-afba-95b33dbe4433\u003c\/tt\u003e) both referencing \u003ctt class=\"remarkup-monospaced\"\u003elocal_group_wikisource_T_parsoid_stash_html\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\" style=\" max-height: 12em; overflow: auto;\"\u003exenon:\/var\/log\/cassandra$ grep -e 958a82f0-6d1d-11e5-b81f-81302e0f25c7 -e 957dffd0-6d1d-11e5-afba-95b33dbe4433 system.log\nINFO  [SharedPool-Worker-1] 2015-10-07 18:02:33,504 MigrationManager.java:249 - Create new ColumnFamily: org.apache.cassandra.config.CFMetaData@61182de5[cfId=958a82f0-6d1d-11e5-b81f-81302e0f25c7,ksName=local_group_wikisource_T_parsoid_stash_html,cfName=meta,cfType=Standard,comparator=org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType),org.apache.cassandra.db.marshal.UTF8Type),comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=_del, type=org.apache.cassandra.db.marshal.TimeUUIDType, kind=REGULAR, componentIndex=1, indexName=null, indexType=null}, ColumnDefinition{name=value, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=1, indexName=null, indexType=null}, ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type, kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=tid, type=org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType), kind=CLUSTERING_COLUMN, componentIndex=0, indexName=null, indexType=null}],compactionStrategyClass=class org.apache.cassandra.db.compaction.LeveledCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.1,memtableFlushPeriod=0,caching={"keys":"ALL", "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=false]\nINFO  [MigrationStage:1] 2015-10-07 18:02:33,664 DefsTables.java:373 - Loading org.apache.cassandra.config.CFMetaData@4d3c59ec[cfId=957dffd0-6d1d-11e5-afba-95b33dbe4433,ksName=local_group_wikisource_T_parsoid_stash_html,cfName=meta,cfType=Standard,comparator=org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType),org.apache.cassandra.db.marshal.UTF8Type),comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=_del, type=org.apache.cassandra.db.marshal.TimeUUIDType, kind=REGULAR, componentIndex=1, indexName=null, indexType=null}, ColumnDefinition{name=value, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=1, indexName=null, indexType=null}, ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type, kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=tid, type=org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType), kind=CLUSTERING_COLUMN, componentIndex=0, indexName=null, indexType=null}],compactionStrategyClass=class org.apache.cassandra.db.compaction.LeveledCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.1,memtableFlushPeriod=0,caching={"keys":"ALL", "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=false]\njava.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\njava.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: java.util.concurrent.ExecutionException: java.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: java.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\njava.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: java.util.concurrent.ExecutionException: java.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: java.lang.RuntimeException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\nCaused by: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch (found 958a82f0-6d1d-11e5-b81f-81302e0f25c7; expected 957dffd0-6d1d-11e5-afba-95b33dbe4433)\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_688\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1715930\" id=\"1715930\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_687\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1715930\" data-sigil=\"has-tooltip\" data-meta=\"0_686\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2015, 6:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-09 18:29:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_684\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_685\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_183\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI think I encountered a similar problem after accidentally bootstrapping a node off itself in the past (by having it listed as its own seed). IIRC, nuking that node's data (the equivalent of rm -rf \/var\/lib\/cassandra\/*) and re-bootstrapping it fixed it.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_697\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1747776\" id=\"1747776\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_696\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_184\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1747776\" data-sigil=\"has-tooltip\" data-meta=\"0_695\"\u003e\u003cspan class=\"screen-only\"\u003eOct 23 2015, 11:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-23 11:19:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_693\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_694\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_185\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003erestbase-test2002 is being converted to multi instance, currently bootstrapping \u003ctt class=\"remarkup-monospaced\"\u003eb\u003c\/tt\u003e instance:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ nodetool status -r\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eDatacenter: codfw\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e=================\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eStatus=Up\/Down\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e|\/ State=Normal\/Leaving\/Joining\/Moving\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e--  Address                          Load       Tokens  Owns    Host ID                               Rack\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eDL  restbase-test2002.codfw.wmnet    148.94 GB  256     ?       4afd4cc2-fe90-4868-bb99-3bf6496d5a57  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase-test2003.codfw.wmnet    154.26 GB  256     ?       ec1d70ad-aab6-4f68-af44-2bc31b500804  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase-test2001-a.codfw.wmnet  136.92 GB  256     ?       3d5e1cbf-495d-4d48-a712-e489c909ff8a  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase-test2001-b.codfw.wmnet  117.28 GB  256     ?       11661295-f159-4fb8-8830-38a4ebc150cf  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUJ  restbase-test2002-b.codfw.wmnet  13.67 GB   256     ?       2eb6707c-f904-4d1b-8e52-c63c96232f7f  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eDatacenter: eqiad\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e=================\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eStatus=Up\/Down\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e|\/ State=Normal\/Leaving\/Joining\/Moving\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e--  Address                          Load       Tokens  Owns    Host ID                               Rack\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  cerium.eqiad.wmnet               165.52 GB  256     ?       235934bc-33f0-4744-b374-f7c96c8edc95  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  praseodymium.eqiad.wmnet         165.05 GB  256     ?       5be66290-4784-4fed-a4f1-d496ea09d803  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  xenon.eqiad.wmnet                164.46 GB  256     ?       20ec6bb4-195b-479c-a540-e6a7c70b589d  rack1\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eNote: Non-system keyspaces don't have the same replication settings, effective ownership information is meaningless\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003enote that \u003ctt class=\"remarkup-monospaced\"\u003erestbase-test2002\u003c\/tt\u003e is shown as 'leaving', that's the effect of running \u003ctt class=\"remarkup-monospaced\"\u003enodetool removenode\u003c\/tt\u003e after reimaging as opposed to \u003ctt class=\"remarkup-monospaced\"\u003enodetool decomission\u003c\/tt\u003e before reimaging which I missed\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_706\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1757846\" id=\"1757846\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_705\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_186\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1757846\" data-sigil=\"has-tooltip\" data-meta=\"0_704\"\u003e\u003cspan class=\"screen-only\"\u003eOct 27 2015, 4:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-27 16:12:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_702\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_703\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_187\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 249152 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncassandra: add eqiad test cluster multiple instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/249152\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/249152\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_715\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1757894\" id=\"1757894\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_188\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1757894\" data-sigil=\"has-tooltip\" data-meta=\"0_713\"\u003e\u003cspan class=\"screen-only\"\u003eOct 27 2015, 4:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-27 16:20:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_711\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_712\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 249152 merged by Filippo Giunchedi:\u003cbr \/\u003e\ncassandra: add eqiad test cluster multiple instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/249152\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/249152\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_718\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1766937\" id=\"1766937\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_717\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_190\"\u003eEevans\u003c\/a\u003e mentioned this in \u003ca href=\"\/T117114\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003eT117114: Ensure ansible-deploy can cope with multi-instance restarts\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1766937\" data-sigil=\"has-tooltip\" data-meta=\"0_716\"\u003e\u003cspan class=\"screen-only\"\u003eOct 29 2015, 10:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-29 22:06:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_721\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1766975\" id=\"1766975\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_720\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_192\"\u003eEevans\u003c\/a\u003e added a subtask: \u003ca href=\"\/T117114\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_193\"\u003eT117114: Ensure ansible-deploy can cope with multi-instance restarts\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1766975\" data-sigil=\"has-tooltip\" data-meta=\"0_719\"\u003e\u003cspan class=\"screen-only\"\u003eOct 29 2015, 10:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-10-29 22:15:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_724\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1775961\" id=\"1775961\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_723\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003eEevans\u003c\/a\u003e closed subtask \u003ca href=\"\/T117114\" class=\"phui-handle handle-status-closed\"\u003eT117114: Ensure ansible-deploy can cope with multi-instance restarts\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1775961\" data-sigil=\"has-tooltip\" data-meta=\"0_722\"\u003e\u003cspan class=\"screen-only\"\u003eNov 2 2015, 11:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-02 23:19:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_733\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1777165\" id=\"1777165\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_732\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_195\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1777165\" data-sigil=\"has-tooltip\" data-meta=\"0_731\"\u003e\u003cspan class=\"screen-only\"\u003eNov 3 2015, 2:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-03 14:01:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_729\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_730\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003exenon reimaged today, currently bootstrapping one instance\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003exenon:~$ nodetool-a netstats | grep "bytes total"\n        Receiving 216 files, 202613100728 bytes total. Already received 80 files, 44656735084 bytes total\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_742\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1777454\" id=\"1777454\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_741\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_197\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1777454\" data-sigil=\"has-tooltip\" data-meta=\"0_740\"\u003e\u003cspan class=\"screen-only\"\u003eNov 3 2015, 3:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-03 15:39:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_738\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_739\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 234292 abandoned by Filippo Giunchedi:\u003cbr \/\u003e\nWIP: xenon additional instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/234292\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/234292\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_751\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1777455\" id=\"1777455\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_750\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_199\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1777455\" data-sigil=\"has-tooltip\" data-meta=\"0_749\"\u003e\u003cspan class=\"screen-only\"\u003eNov 3 2015, 3:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-03 15:39:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_747\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_748\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_200\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 234286 abandoned by Filippo Giunchedi:\u003cbr \/\u003e\nxenon additional instances\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/234286\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/234286\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_760\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781117\" id=\"1781117\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_759\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_201\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781117\" data-sigil=\"has-tooltip\" data-meta=\"0_758\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 9:55 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 09:55:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_756\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_757\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003exenon-b bootstrapping\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ nodetool-b netstats | grep "bytes total"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e        Receiving 53 files, 49753415567 bytes total. Already received 18 files, 5097689040 bytes total\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e        Receiving 214 files, 46490176224 bytes total. Already received 78 files, 5188120899 bytes total\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e        Receiving 52 files, 46954589749 bytes total. Already received 14 files, 5073911653 bytes total\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_769\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781721\" id=\"1781721\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_768\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_203\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781721\" data-sigil=\"has-tooltip\" data-meta=\"0_767\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 2:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 14:20:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_765\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_766\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_204\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eas it stands however there isn't enough space on eqiad test cluster to allocate two instances (I've stopped bootstrapping \u003ctt class=\"remarkup-monospaced\"\u003exenon-b\u003c\/tt\u003e). one solution would be to trim data from the test cluster\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003exenon:~$ nodetool-a status -r\nDatacenter: codfw\n=================\nStatus=Up\/Down\n|\/ State=Normal\/Leaving\/Joining\/Moving\n--  Address                          Load       Tokens  Owns    Host ID                               Rack\nUN  restbase-test2001-a.codfw.wmnet  100.68 GB  256     ?       3d5e1cbf-495d-4d48-a712-e489c909ff8a  rack1\nUN  restbase-test2001-b.codfw.wmnet  132.09 GB  256     ?       11661295-f159-4fb8-8830-38a4ebc150cf  rack1\nUN  restbase-test2002-a.codfw.wmnet  93.74 GB   256     ?       63424769-1155-4c06-a83f-3e8cab988f20  rack1\nUN  restbase-test2002-b.codfw.wmnet  142.38 GB  256     ?       2eb6707c-f904-4d1b-8e52-c63c96232f7f  rack1\nUN  restbase-test2003-a.codfw.wmnet  108.13 GB  256     ?       2f443acf-20ed-423f-bdce-26268796c3a5  rack1\nUN  restbase-test2003-b.codfw.wmnet  96.81 GB   256     ?       ab2fbf49-a4f3-4ae9-976c-4d41abf50b67  rack1\nDatacenter: eqiad\n=================\nStatus=Up\/Down\n|\/ State=Normal\/Leaving\/Joining\/Moving\n--  Address                          Load       Tokens  Owns    Host ID                               Rack\nUN  cerium.eqiad.wmnet               193.44 GB  256     ?       235934bc-33f0-4744-b374-f7c96c8edc95  rack1\nUN  praseodymium.eqiad.wmnet         193.5 GB   256     ?       5be66290-4784-4fed-a4f1-d496ea09d803  rack1\nUN  xenon-a.eqiad.wmnet              193.42 GB  256     ?       0d691414-4132-4854-a00d-1d2671e15728  rack1\n\nNote: Non-system keyspaces don't have the same replication settings, effective ownership information is meaningless\nxenon:~$ df -h\nFilesystem                 Size  Used Avail Use% Mounted on\nudev                        10M     0   10M   0% \/dev\ntmpfs                      3.2G   17M  3.2G   1% \/run\n\/dev\/md0                    28G  1.9G   25G   8% \/\ntmpfs                      7.8G     0  7.8G   0% \/dev\/shm\ntmpfs                      5.0M     0  5.0M   0% \/run\/lock\ntmpfs                      7.8G     0  7.8G   0% \/sys\/fs\/cgroup\n\/dev\/mapper\/xenon--vg-srv  355G  313G   25G  93% \/srv\nxenon:~$\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_778\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781831\" id=\"1781831\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_777\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_205\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781831\" data-sigil=\"has-tooltip\" data-meta=\"0_776\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 2:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 14:50:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_774\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_775\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_206\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1781721\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1781721\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eas it stands however there isn't enough space on eqiad test cluster to allocate two instances (I've stopped bootstrapping \u003ctt class=\"remarkup-monospaced\"\u003exenon-b\u003c\/tt\u003e). one solution would be to trim data from the test cluster\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eOnce bootstrapped, we could \u003ctt class=\"remarkup-monospaced\"\u003enodetool clean\u003c\/tt\u003e the other instance and bring the space back down, but this does create a chicken-egg scenario, doesn't it?  \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e: Would it save any time to convert praseodymium and\/or cerium to one instance while we figure out how to proceed here?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_787\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781842\" id=\"1781842\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_786\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_207\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781842\" data-sigil=\"has-tooltip\" data-meta=\"0_785\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 2:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 14:54:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_783\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_784\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_208\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eyeah we can proceed with praseodymium and cerium meanwhile to save time. I don't think it'd be able to finish bootstrapping though? 2x data is still > 355G\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_796\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781844\" id=\"1781844\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_795\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_209\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781844\" data-sigil=\"has-tooltip\" data-meta=\"0_794\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 14:57:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_792\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_793\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_210\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1781842\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1781842\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eyeah we can proceed with praseodymium and cerium meanwhile to save time. I don't think it'd be able to finish bootstrapping though? 2x data is still > 355G\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eSorry, I meant, you could reimage them and bootstrap a single instance (basically, bring them to the same state as xenon is currently).  The additional instances could be bootstrapped in place when the disk usage is sorted.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_805\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781920\" id=\"1781920\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_804\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_211\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781920\" data-sigil=\"has-tooltip\" data-meta=\"0_803\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 3:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 15:45:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_801\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_802\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_212\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAlternatively, wouldn't rm -rf \/var\/lib\/cassandra\/* clear up the space as well?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_814\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781940\" id=\"1781940\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_813\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_213\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781940\" data-sigil=\"has-tooltip\" data-meta=\"0_812\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 3:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 15:50:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_810\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_811\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_214\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1781920\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1781920\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAlternatively, wouldn't rm -rf \/var\/lib\/cassandra\/* clear up the space as well?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003efrom xenon? no, the machine has been reinstalled\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_823\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781948\" id=\"1781948\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_822\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_215\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1781948\" data-sigil=\"has-tooltip\" data-meta=\"0_821\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 3:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 15:51:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_819\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_820\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_216\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1781920\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1781920\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAlternatively, wouldn't rm -rf \/var\/lib\/cassandra\/* clear up the space as well?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eCan you expound on this?\u003c\/p\u003e\n\n\u003cp\u003exenon was decomissioned, reimaged, and bootstrapped as xenon-a.eqiad.wmnet.  Afterward, it contained 1\/3 of the eqiad data (as it did before).  Bootstrapping xenon-b.eqiad.wmnet entails copying an additional 1\/4 of the eqiad data onto xenon.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_833\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1781977\" id=\"1781977\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_832\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_217\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1781977\" data-sigil=\"has-tooltip\" data-meta=\"0_831\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 3:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 15:58:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_829\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_830\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_218\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003efrom xenon?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI was thinking of praseodymium and cerium, but re-imaging is probably desirable anyway. We should look into increasing the storage space on these test systems, or migrating to different spares. For now, do you think we could move to codfw production after re-imaging praseodymium \/ cerium with a single multi-instance setup?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_842\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1782049\" id=\"1782049\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_841\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_219\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1782049\" data-sigil=\"has-tooltip\" data-meta=\"0_840\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 4:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 16:08:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_838\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_839\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_220\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1781977\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1781977\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003efrom xenon?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI was thinking of praseodymium and cerium, but re-imaging is probably desirable anyway. We should look into increasing the storage space on these test systems, or migrating to different spares.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003ewe could get bigger SSDs for those too, how would we go about trimming space if need be?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eFor now, do you think we could move to codfw production after re-imaging praseodymium \/ cerium with a single multi-instance setup?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003ewe could, however the closer the test cluster is to production the better\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_852\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1782090\" id=\"1782090\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_851\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_221\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1782090\" data-sigil=\"has-tooltip\" data-meta=\"0_850\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 4:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 16:15:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_848\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_849\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_222\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003ewe could get bigger SSDs for those too,\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThat might be worth it. I wonder if the RMA for the original batch of 850s came back already.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003ehow would we go about trimming space if need be?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe'd need to delete data. The quickest way would be to drop all wikipedia keyspaces, and have them be re-created from scratch. However, that would clear basically all data, and re-creating that will take a couple of days. Alternatively, we could delete a part of the data. For this, we'd have to select some (by token range, for example), delete it, and then wait for compaction to actually clear the space.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003ewe could, however the closer the test cluster is to production the better\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgreed. I'm just wondering if it's worth blocking on having multiple instances on the small nodes in the staging cluster while we figure out storage size \/ deletion.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_861\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oufhulum4kze5rgdza3a\/PHID-FILE-bt7yrtg7nbh3gooznjl2\/profile-me.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mobrovac\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1782118\" id=\"1782118\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_860\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mobrovac\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_223\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e mobrovac\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1782118\" data-sigil=\"has-tooltip\" data-meta=\"0_859\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 4:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 16:23:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_857\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_858\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_224\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1782090\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1782090\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eWe'd need to delete data. The quickest way would be to drop all wikipedia keyspaces, and have them be re-created from scratch. However, that would clear basically all data, and re-creating that will take a couple of days.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWhy would we need to wait for all of the data to be back there? Can't we drop all keyspaces, recreate them, move all of the nodes to multi-instances and then use the dumper script to get back all of the content? That could also be an opportunity to watch multi-instances at work.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003ewe could, however the closer the test cluster is to production the better\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAgreed. I'm just wondering if it's worth blocking on having multiple instances on the small nodes in the staging cluster while we figure out storage size \/ deletion.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI'd prefer to have a working staging environment before proceeding to production, honestly. While we do want to get it out in prod ASAP, I doubt that a couple of days will make any difference, while at the same time giving us a chance to make sure all's good.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_870\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1782155\" id=\"1782155\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_869\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_225\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1782155\" data-sigil=\"has-tooltip\" data-meta=\"0_868\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 4:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 16:29:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_866\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_867\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_226\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWhy would we need to wait for all of the data to be back there?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI never said we'd need to wait. However, no data makes for less realistic bootstrap testing.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI'd prefer to have a working staging environment before proceeding to production, honestly.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eNo disagreement here either. The question wasn't about that though, but about having more than one multi-instance per node on the smaller staging cluster nodes right away. Basically, are we okay with having -a only on the small nodes until we can expand the capacity, or should we make sure to have the same number on all nodes right away.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_879\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ckuog6lpgvwxciyg5wab\/PHID-FILE-sfqnl3dgxxdvoa527rb4\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Eevans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1782369\" id=\"1782369\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_878\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Eevans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_227\"\u003eEevans\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1782369\" data-sigil=\"has-tooltip\" data-meta=\"0_877\"\u003e\u003cspan class=\"screen-only\"\u003eNov 4 2015, 5:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-04 17:07:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_875\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_876\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_228\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1782049\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1782049\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003ewe could get bigger SSDs for those too\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eFWIW, it \u003cem\u003emight\u003c\/em\u003e be enough to get the disk space up on one hardware node.  Once one of those nodes has two instances, the per-instance share will drop, (and we can do cleanups behind us as we go).  Back-of-napkin: If the per-instance share is ~193G at 3 nodes, then it should be ~144G @ 4, and setting aside overhead for the moment, we'd then need ~288G per host thereafter.\u003c\/p\u003e\n\n\u003cp\u003eObviously, long-term you'd want storage to be homogeneous, but maybe as a near-term fix this would be easier.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T95253#1782155\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT95253#1782155\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eWhy would we need to wait for all of the data to be back there?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI never said we'd need to wait. However, no data makes for less realistic bootstrap testing.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eI'd prefer to have a working staging environment before proceeding to production, honestly.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eNo disagreement here either. The question wasn't about that though, but about having more than one multi-instance per node on the smaller staging cluster nodes right away. Basically, are we okay with having -a only on the small nodes until we can expand the capacity, or should we make sure to have the same number on all nodes right away.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eSeems kind of a toss up TBH; The data set in staging isn't anywhere near representative of production as it is, I don't know that making it smaller makes for better or worse testing than running fewer (i.e. one) instances (we did bootstrap the full data size 6 times in codfw, which is something).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_888\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1795557\" id=\"1795557\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_887\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_229\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1795557\" data-sigil=\"has-tooltip\" data-meta=\"0_886\"\u003e\u003cspan class=\"screen-only\"\u003eNov 10 2015, 12:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-10 12:55:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_884\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_885\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_230\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've started \u003ctt class=\"remarkup-monospaced\"\u003enodetool decommission\u003c\/tt\u003e yesterday morning on restbase2001, still going\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003erestbase2001:\/var\/log\/cassandra$ nodetool netstats | grep 'bytes total'\n        Sending 560 files, 504539572254 bytes total. Already sent 333 files, 408117717136 bytes total\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_891\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1798099\" id=\"1798099\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_890\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_231\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e reopened subtask \u003ca href=\"\/T117114\" class=\"phui-handle handle-status-closed\"\u003eT117114: Ensure ansible-deploy can cope with multi-instance restarts\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1798099\" data-sigil=\"has-tooltip\" data-meta=\"0_889\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2015, 12:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-11 00:14:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_900\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1798940\" id=\"1798940\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_899\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_232\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1798940\" data-sigil=\"has-tooltip\" data-meta=\"0_898\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2015, 12:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-11 12:58:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_896\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_897\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_233\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 252426 had a related patch set uploaded (by Filippo Giunchedi):\u003cbr \/\u003e\ncassandra: add restbase2001 instance\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/252426\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/252426\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_909\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1798994\" id=\"1798994\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_908\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_234\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1798994\" data-sigil=\"has-tooltip\" data-meta=\"0_907\"\u003e\u003cspan class=\"screen-only\"\u003eNov 11 2015, 2:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-11 14:05:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_905\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_906\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_235\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 252426 merged by Filippo Giunchedi:\u003cbr \/\u003e\ncassandra: add restbase2001 instance\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/252426\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/252426\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_918\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1807247\" id=\"1807247\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_917\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_236\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1807247\" data-sigil=\"has-tooltip\" data-meta=\"0_916\"\u003e\u003cspan class=\"screen-only\"\u003eNov 16 2015, 9:34 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-16 09:34:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_914\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_915\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_237\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003erestbase2001-b\u003c\/tt\u003e has finished bootstrapping (times UTC) and joined the cluster (cql interface up)\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e15:35 -icinga-wm:#wikimedia-operations- PROBLEM - cassandra-b CQL 10.192.16.163:9042 on restbase2001 is CRITICAL: Connection refused\n--- Day changed Sat Nov 14 2015\n04:08 -icinga-wm:#wikimedia-operations- RECOVERY - cassandra-b CQL 10.192.16.163:9042 on restbase2001 is OK: TCP OK - 0.036 second response time on port 9042\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_927\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1828490\" id=\"1828490\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_926\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_238\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1828490\" data-sigil=\"has-tooltip\" data-meta=\"0_925\"\u003e\u003cspan class=\"screen-only\"\u003eNov 24 2015, 5:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-24 17:31:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_923\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_924\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_239\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've started decommissioning \u003ctt class=\"remarkup-monospaced\"\u003erestbase1001\u003c\/tt\u003e on monday, though there isn't enough space on \u003ctt class=\"remarkup-monospaced\"\u003erestbase1007\u003c\/tt\u003e to hold half data from \u003ctt class=\"remarkup-monospaced\"\u003erestbase1001\u003c\/tt\u003e, thus we've interrupted the decommissioning\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\" style=\" max-height: 12em; overflow: auto;\"\u003e\u003cspan class=\"gp\"\u003e$ nodetool status -r\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eDatacenter: codfw\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e=================\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eStatus=Up\/Down\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e|\/ State=Normal\/Leaving\/Joining\/Moving\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e--  Address                     Load       Tokens  Owns    Host ID                               Rack\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2001-a.codfw.wmnet  688.24 GB  256     ?       7a071b47-a5cb-4051-bd80-c9a6b11c469a  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2001-b.codfw.wmnet  512.43 GB  256     ?       2acb7c90-c929-44a0-baa2-c50526cb08a5  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2001-c.codfw.wmnet  417.9 GB   256     ?       dac7667c-05e4-4c75-8ccd-a27e56b5c614  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2005.codfw.wmnet    716.83 GB  256     ?       dec9c52c-9327-4707-aabd-cb8eb1f7cb21  d\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2002-a.codfw.wmnet  463.56 GB  256     ?       9c67467f-3e48-436c-8ee3-b3d44816a7e5  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2002-b.codfw.wmnet  297.18 GB  256     ?       3c5fdd02-d97c-4385-8ed4-3ee578b75255  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2006.codfw.wmnet    697.86 GB  256     ?       17291060-65d0-4096-a6a9-9d193fe1256d  d\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2002-c.codfw.wmnet  223.94 GB  256     ?       f902b388-6143-4ad9-99e5-a379d11ac315  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2003.codfw.wmnet    688.14 GB  256     ?       80dfa7d8-8478-4c08-b104-545f702f40e9  c\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase2004.codfw.wmnet    721.14 GB  256     ?       7baf3975-7450-4d9a-9daf-d8fe6141ff0a  c\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eDatacenter: eqiad\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e=================\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eStatus=Up\/Down\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e|\/ State=Normal\/Leaving\/Joining\/Moving\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e--  Address                     Load       Tokens  Owns    Host ID                               Rack\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1004.eqiad.wmnet    891.47 GB  256     ?       798ff758-8c91-46e0-b85e-dad356c46f20  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1008.eqiad.wmnet    1.03 TB    256     ?       e9ab408a-309e-4e93-b145-9ac5f2365523  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1005.eqiad.wmnet    922.27 GB  256     ?       325e01e8-debe-45f0-a8c2-93b3baa58968  d\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1006.eqiad.wmnet    882.3 GB   256     ?       2abf437d-a16d-406b-a6de-8d28b7dda808  d\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1001.eqiad.wmnet    1.02 TB    256     ?       c021a198-b7f1-4dc2-94d7-9cb8b8a8df28  a\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1002.eqiad.wmnet    1.3 TB     256     ?       fc041cc8-cd28-4030-b29a-05b9a632cafc  a\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1009.eqiad.wmnet    1.04 TB    256     ?       3ee62592-ef41-445a-863f-b05be9ae5ac8  d\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1003.eqiad.wmnet    917.29 GB  256     ?       88d9ef9f-d81b-466e-babf-6a283b13f648  b\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eUN  restbase1007.eqiad.wmnet    1.24 TB    256     ?       c1b5a012-4840-4096-9a71-ce4d3afb0029  a\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eNote: Non-system keyspaces don't have the same replication settings, effective ownership information is meaningless\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\" style=\" max-height: 12em; overflow: auto;\"\u003erestbase1007:~$ df -h\nFilesystem                        Size  Used Avail Use% Mounted on\nudev                               10M     0   10M   0% \/dev\ntmpfs                             6.3G  281M  6.0G   5% \/run\n\/dev\/md0                           28G   20G  6.0G  78% \/\ntmpfs                              16G     0   16G   0% \/dev\/shm\ntmpfs                             5.0M     0  5.0M   0% \/run\/lock\ntmpfs                              16G     0   16G   0% \/sys\/fs\/cgroup\n\/dev\/mapper\/restbase1007--vg-var  1.8T  1.7T   70G  96% \/var\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ewe'll need to reduce data until it'd be able to hold a node decommissioning\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_937\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1829601\" id=\"1829601\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_936\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_240\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#1829601\" data-sigil=\"has-tooltip\" data-meta=\"0_935\"\u003e\u003cspan class=\"screen-only\"\u003eNov 24 2015, 8:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-24 20:50:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_933\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_934\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_241\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSome notes:\u003c\/p\u003e\n\n\u003cp\u003eWe should be able to get the data on 1007 down to ~950G by waiting for the right moment in the compaction cycle, and then throttling \/ disabling compactions temporarily. I think this is still too tight, though.\u003c\/p\u003e\n\n\u003cp\u003eI wonder if we could get away without decommissioning & the associated data move by copying \/var\/lib\/cassandra\/* on 1001 to another node (rsync, then final catch-up rsync after shutting down Cassandra), re-imaging 1001 without decommissioning it, and then copying the data into the new location, and restarting the new instance using it. To avoid a major impact on read operations, we could temporarily set the read consistency to \u003ctt class=\"remarkup-monospaced\"\u003eone\u003c\/tt\u003e instead of \u003ctt class=\"remarkup-monospaced\"\u003elocalQuorum\u003c\/tt\u003e. While not ideal either because of the reduced redundancy and consistency, it might be a way to get this done without having to delete data.\u003c\/p\u003e\n\n\u003cp\u003eAnother idea would be to add another SSD to at least 1007, and expand the filesystem to use it. \u003ca href=\"\/p\/RobH\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@RobH\u003c\/span\u003e\u003c\/a\u003e: Do we have any Samsung 1T SSD spares in eqiad?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_940\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"1832398\" id=\"1832398\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_939\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_242\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e mentioned this in \u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_243\"\u003eUnknown Object (Task)\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1832398\" data-sigil=\"has-tooltip\" data-meta=\"0_938\"\u003e\u003cspan class=\"screen-only\"\u003eNov 25 2015, 8:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-25 20:47:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_949\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"1833191\" id=\"1833191\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_948\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_244\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#1833191\" data-sigil=\"has-tooltip\" data-meta=\"0_947\"\u003e\u003cspan class=\"screen-only\"\u003eNov 26 2015, 2:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2015-11-26 02:11:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_945\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_946\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_245\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e: I created T119659 to get SSDs for 100[7-9], which should let us move forward without running out of space.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-u7udgblfyop6qd5wxot6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-uw3bqy62pfnqgycfnli2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-acxz2a7hhyfbbq6z7clz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-wggeslexgkhlzqvxygvo"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-cwrzfzaz26vpanpuntbm"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-mszihytuo3ij3fcxcxgm"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-pk26wyzixalyggv"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"phid":"PHID-XACT-TASK-rc6v6ciuim2e6l7"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-6qy64xq5hqosz4l"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-rjhqoptz3iojmks"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-srlzcmsvnqihhyy"},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"phid":"PHID-XACT-TASK-wfpefnmnxwn2brg"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-4dolyd6grvfzgrc"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-no5oa3k4kgbn2ik44sib"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-pegtsjlt4mfjhv6"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-zetrkfrxgreftdzubwyz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-eo3y3expzmzi3fv"},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"phid":"PHID-XACT-TASK-ltwsr2s2mm42hj4"},{"hovercardSpec":{"objectPHID":"PHID-USER-xwxce3kpvsv54t735g7q"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-rhj2ivps7licc4h"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-7itgkheuk3aebkf"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-yyrmh5idmdgcjli"},{"phid":"PHID-XACT-TASK-sfli7pbnov5on6x"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-yumvheq5gvzt7ym"},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"phid":"PHID-XACT-TASK-rialdykxredpwv6"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-ef3wo5lqonthply"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-3toq3ioxmj4ydls"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-g7eocut7irllvpi"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-cjzutz5gvqeqt4nh7owd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-j2m63d4s5ca5o7l"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-ih7tqnh33jir6rz"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-nqp2nti3zxzabrj"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-aql27qi2zdsafey"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ht3pa2ys6snexgs"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-7dzk2plcqorisce"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-jmofzhm3nee4of6"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-uwonom6hyfnq27o"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-y6pz6mlxy3iudky"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-42bbixlekl2jckr7eqfe"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-gkgtqsrysuhw772"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-pqz5hwm6ogxtdsj"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-vibncjhtmrehehu"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-2orf2kc4whob2embbmnj"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-6wp5d5bxelnnkyr"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-uf2gonpghqgbzmw"},{"hovercardSpec":{"objectPHID":"PHID-USER-oxd6f6xemkuyttw7z7wl"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-3orlef7lqithbzd"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-6bepmg5nrtpfic5d2uip"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-um5pjri5gaenjgo"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-232lx7fgfgjj6rgogbdj"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-66hnm7h4oot3zpa"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-wxxmkfk5zvapici"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-c5yygtfitjkahwuldsxi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-mczdndgpk7pnd3t"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-n7fodr42rwijpsz"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-qevtvdxmt7q3cig"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-pzo3lpd5oltjftn"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-wtkktndwf5jugwy"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-ybnjwtqakat7dgt"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-cuw4quythxha32a"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-fj5rnxhyt7wsmpl"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yml746unbbn7wih7euan"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yml746unbbn7wih7euan"}},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-wpvpfc5oa6cigxu"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-73fsdbwz646v5hu"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-sbh5bpapxmlat6m"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-gxwzyxuo7h5xyw5"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-ubci3dlcittpsqa"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-2zd26mxwpgw6wjx"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-eubtdrnbkegmcpq"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-xljtzesg3c6vufw"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-3ll2776zeyzyjzd"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-ljz7cwr7wdg5qtd"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-qwivnvllj3isbcr"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-q4cu5uj6w3zn2ev"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-zvv2aqt7miiaaxj"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-ig5dbctetkchtyr"},{"hovercardSpec":{"objectPHID":"PHID-USER-hxwwywcyzpooynxuo7a2"}},{"phid":"PHID-XACT-TASK-ro2is6iqq6zfgij"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-c22zantu63u5jdv"},{"hovercardSpec":{"objectPHID":"PHID-USER-7mtggupugbd5i55vuvdc"}},{"phid":"PHID-XACT-TASK-p7nzz6jwrbt2lhp"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-sl4bjvyotca6o6f"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-pdry3v6mmjgwch5"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-silo7itujlypclm"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-y6sjydgiblyyzke"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-v4ayfgws5qhbebz"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-ebn2csxtolg7e43"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-emchentrrcoofwseerrm"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-wimvou7t5vt6o72"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-zefln7cp2ww3g3u","anchor":"1410256"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sdwuzaoklo4pnv4","anchor":"1410258"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pk26wyzixalyggv\/","ref":"T95253#1452113"},[],{"anchor":"1452113"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pk26wyzixalyggv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_254\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pk26wyzixalyggv","anchor":"1452113"},{"tip":"Via Web"},[],[],[],{"phid":"PHID-XACT-TASK-rowqzr5ddm6yz23","anchor":"1452125"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z4x2togqd4uepoq","anchor":"1452134"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-rc6v6ciuim2e6l7\/","ref":"T95253#1453721"},[],{"anchor":"1453721"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_269\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_270\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rc6v6ciuim2e6l7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_271\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_272\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rc6v6ciuim2e6l7","anchor":"1453721"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-6qy64xq5hqosz4l\/","ref":"T95253#1454051"},[],{"anchor":"1454051"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_278\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_279\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6qy64xq5hqosz4l\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_280\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-6qy64xq5hqosz4l\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6qy64xq5hqosz4l","anchor":"1454051"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-rjhqoptz3iojmks\/","ref":"T95253#1454385"},[],{"anchor":"1454385"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_288\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rjhqoptz3iojmks\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_290\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rjhqoptz3iojmks","anchor":"1454385"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vccb7acddgagker","anchor":"1454432"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rakziltd7aheemy","anchor":"1454533"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-srlzcmsvnqihhyy\/","ref":"T95253#1465308"},[],{"anchor":"1465308"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_303\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_304\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-srlzcmsvnqihhyy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_305\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_306\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-srlzcmsvnqihhyy","anchor":"1465308"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-wfpefnmnxwn2brg\/","ref":"T95253#1465354"},[],{"anchor":"1465354"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_312\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_313\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wfpefnmnxwn2brg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_314\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_315\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wfpefnmnxwn2brg","anchor":"1465354"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-4dolyd6grvfzgrc\/","ref":"T95253#1465372"},[],{"anchor":"1465372"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_321\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_322\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4dolyd6grvfzgrc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_323\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_324\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-4dolyd6grvfzgrc\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4dolyd6grvfzgrc","anchor":"1465372"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2f7mm6eior6cerp","anchor":"1467127"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pegtsjlt4mfjhv6\/","ref":"T95253#1524978"},[],{"anchor":"1524978"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_334\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_335\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pegtsjlt4mfjhv6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_336\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pegtsjlt4mfjhv6","anchor":"1524978"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4m7xg4lyroffrey","anchor":"1525011"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-itj3fte5yqrsixs","anchor":"1525099"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-eo3y3expzmzi3fv\/","ref":"T95253#1527183"},[],{"anchor":"1527183"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_349\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eo3y3expzmzi3fv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_351\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eo3y3expzmzi3fv","anchor":"1527183"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ltwsr2s2mm42hj4\/","ref":"T95253#1527327"},[],{"anchor":"1527327"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_358\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ltwsr2s2mm42hj4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_360\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_361\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ltwsr2s2mm42hj4","anchor":"1527327"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5a4adzbyhtck57z","anchor":"1527572"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-rhj2ivps7licc4h\/","ref":"T95253#1532040"},[],{"anchor":"1532040"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_370\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rhj2ivps7licc4h\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_372\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_373\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-rhj2ivps7licc4h\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rhj2ivps7licc4h","anchor":"1532040"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-7itgkheuk3aebkf\/","ref":"T95253#1535221"},[],{"anchor":"1535221"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_380\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_381\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7itgkheuk3aebkf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_382\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7itgkheuk3aebkf","anchor":"1535221"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-yyrmh5idmdgcjli\/","ref":"T95253#1536539"},[],{"anchor":"1536539"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_389\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yyrmh5idmdgcjli\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_391\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_392\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yyrmh5idmdgcjli","anchor":"1536539"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-sfli7pbnov5on6x\/","ref":"T95253#1538970"},[],{"anchor":"1538970"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_398\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_399\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sfli7pbnov5on6x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_400\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_401\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-sfli7pbnov5on6x","anchor":"1538970"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-bbjp44wyfrextso","anchor":"1538971"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-yumvheq5gvzt7ym\/","ref":"T95253#1539350"},[],{"anchor":"1539350"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_410\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_411\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yumvheq5gvzt7ym\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_412\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_413\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yumvheq5gvzt7ym","anchor":"1539350"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-rialdykxredpwv6\/","ref":"T95253#1539380"},[],{"anchor":"1539380"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_419\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_420\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rialdykxredpwv6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_421\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_422\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rialdykxredpwv6","anchor":"1539380"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ef3wo5lqonthply\/","ref":"T95253#1546025"},[],{"anchor":"1546025"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_428\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_429\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ef3wo5lqonthply\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_430\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ef3wo5lqonthply","anchor":"1546025"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-3toq3ioxmj4ydls\/","ref":"T95253#1546040"},[],{"anchor":"1546040"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_437\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_438\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3toq3ioxmj4ydls\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_439\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3toq3ioxmj4ydls","anchor":"1546040"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-g7eocut7irllvpi\/","ref":"T95253#1546867"},[],{"anchor":"1546867"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_448\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-g7eocut7irllvpi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_450\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-g7eocut7irllvpi","anchor":"1546867"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-d7va5t47tf3753w","anchor":"1548359"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-j2m63d4s5ca5o7l\/","ref":"T95253#1557475"},[],{"anchor":"1557475"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_460\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_461\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j2m63d4s5ca5o7l\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_462\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_463\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-j2m63d4s5ca5o7l","anchor":"1557475"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ih7tqnh33jir6rz\/","ref":"T95253#1557554"},[],{"anchor":"1557554"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_469\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_470\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ih7tqnh33jir6rz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_471\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_472\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ih7tqnh33jir6rz","anchor":"1557554"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-nqp2nti3zxzabrj\/","ref":"T95253#1558891"},[],{"anchor":"1558891"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_478\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_479\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nqp2nti3zxzabrj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_480\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_481\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nqp2nti3zxzabrj","anchor":"1558891"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-aql27qi2zdsafey\/","ref":"T95253#1575788"},[],{"anchor":"1575788"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_487\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_488\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aql27qi2zdsafey\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_489\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_490\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-aql27qi2zdsafey","anchor":"1575788"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ht3pa2ys6snexgs\/","ref":"T95253#1580153"},[],{"anchor":"1580153"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_496\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_497\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ht3pa2ys6snexgs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_498\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_499\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ht3pa2ys6snexgs","anchor":"1580153"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-7dzk2plcqorisce\/","ref":"T95253#1580211"},[],{"anchor":"1580211"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_505\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_506\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7dzk2plcqorisce\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_507\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_508\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7dzk2plcqorisce","anchor":"1580211"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-jmofzhm3nee4of6\/","ref":"T95253#1594988"},[],{"anchor":"1594988"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_514\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_515\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jmofzhm3nee4of6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_516\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_517\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jmofzhm3nee4of6","anchor":"1594988"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-uwonom6hyfnq27o\/","ref":"T95253#1684713"},[],{"anchor":"1684713"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_523\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_524\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uwonom6hyfnq27o\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_525\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_526\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uwonom6hyfnq27o","anchor":"1684713"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-exaprxj6q6cu6a6","anchor":"1684759"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-y6pz6mlxy3iudky\/","ref":"T95253#1689431"},[],{"anchor":"1689431"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_535\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-y6pz6mlxy3iudky\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_537\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_538\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-y6pz6mlxy3iudky","anchor":"1689431"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-oqaxvctz4z6eqok","anchor":"1689442"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-gkgtqsrysuhw772\/","ref":"T95253#1691567"},[],{"anchor":"1691567"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_547\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_548\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gkgtqsrysuhw772\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_549\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_550\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-gkgtqsrysuhw772\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_551\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gkgtqsrysuhw772","anchor":"1691567"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pqz5hwm6ogxtdsj\/","ref":"T95253#1693365"},[],{"anchor":"1693365"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_557\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_558\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pqz5hwm6ogxtdsj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_559\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_560\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pqz5hwm6ogxtdsj","anchor":"1693365"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-vibncjhtmrehehu\/","ref":"T95253#1701518"},[],{"anchor":"1701518"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_566\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_567\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vibncjhtmrehehu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_568\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_569\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-vibncjhtmrehehu","anchor":"1701518"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-t6hwwalwkroz2wj","anchor":"1701519"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-6wp5d5bxelnnkyr\/","ref":"T95253#1702183"},[],{"anchor":"1702183"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_578\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_579\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6wp5d5bxelnnkyr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_580\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_581\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6wp5d5bxelnnkyr","anchor":"1702183"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-uf2gonpghqgbzmw\/","ref":"T95253#1705697"},[],{"anchor":"1705697"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_587\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_588\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uf2gonpghqgbzmw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_589\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_590\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uf2gonpghqgbzmw","anchor":"1705697"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5obzaotir2gvqrd","anchor":"1707815"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-3orlef7lqithbzd\/","ref":"T95253#1708242"},[],{"anchor":"1708242"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_599\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_600\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3orlef7lqithbzd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_601\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_602\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3orlef7lqithbzd","anchor":"1708242"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-cxetfjrbjptvq7c","anchor":"1708244"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-um5pjri5gaenjgo\/","ref":"T95253#1708628"},[],{"anchor":"1708628"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_611\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_612\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-um5pjri5gaenjgo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_613\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_614\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-um5pjri5gaenjgo","anchor":"1708628"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-xz6arhsxyl3uxde","anchor":"1708629"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-66hnm7h4oot3zpa\/","ref":"T95253#1711913"},[],{"anchor":"1711913"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_623\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_624\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-66hnm7h4oot3zpa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_625\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_626\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-66hnm7h4oot3zpa","anchor":"1711913"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-wxxmkfk5zvapici\/","ref":"T95253#1711924"},[],{"anchor":"1711924"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_632\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_633\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wxxmkfk5zvapici\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_634\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_635\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wxxmkfk5zvapici","anchor":"1711924"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6yvjrc2lr2kdp4w","anchor":"1713076"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-mczdndgpk7pnd3t\/","ref":"T95253#1714689"},[],{"anchor":"1714689"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_644\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_645\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mczdndgpk7pnd3t\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_646\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_647\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-mczdndgpk7pnd3t","anchor":"1714689"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-n7fodr42rwijpsz\/","ref":"T95253#1715423"},[],{"anchor":"1715423"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_653\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_654\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-n7fodr42rwijpsz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_655\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_656\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-n7fodr42rwijpsz","anchor":"1715423"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-qevtvdxmt7q3cig\/","ref":"T95253#1715480"},[],{"anchor":"1715480"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_662\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_663\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qevtvdxmt7q3cig\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_664\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_665\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qevtvdxmt7q3cig","anchor":"1715480"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pzo3lpd5oltjftn\/","ref":"T95253#1715483"},[],{"anchor":"1715483"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_671\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_672\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pzo3lpd5oltjftn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_673\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_674\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pzo3lpd5oltjftn","anchor":"1715483"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-wtkktndwf5jugwy\/","ref":"T95253#1715930"},[],{"anchor":"1715930"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_680\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_681\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wtkktndwf5jugwy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_682\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_683\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wtkktndwf5jugwy","anchor":"1715930"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ybnjwtqakat7dgt\/","ref":"T95253#1747776"},[],{"anchor":"1747776"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_689\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_690\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ybnjwtqakat7dgt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_691\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_692\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ybnjwtqakat7dgt","anchor":"1747776"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-cuw4quythxha32a\/","ref":"T95253#1757846"},[],{"anchor":"1757846"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_698\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_699\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cuw4quythxha32a\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_700\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_701\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cuw4quythxha32a","anchor":"1757846"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-fj5rnxhyt7wsmpl\/","ref":"T95253#1757894"},[],{"anchor":"1757894"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_707\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_708\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fj5rnxhyt7wsmpl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_709\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_710\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fj5rnxhyt7wsmpl","anchor":"1757894"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bqc5tks5lv6zuvt","anchor":"1766937"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hnr5vnovxgdzxxg","anchor":"1766975"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u5de3b3x6k5yh66","anchor":"1775961"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-wpvpfc5oa6cigxu\/","ref":"T95253#1777165"},[],{"anchor":"1777165"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_725\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_726\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wpvpfc5oa6cigxu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_727\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_728\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wpvpfc5oa6cigxu","anchor":"1777165"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-73fsdbwz646v5hu\/","ref":"T95253#1777454"},[],{"anchor":"1777454"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_734\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_735\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-73fsdbwz646v5hu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_736\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_737\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-73fsdbwz646v5hu","anchor":"1777454"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-sbh5bpapxmlat6m\/","ref":"T95253#1777455"},[],{"anchor":"1777455"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_743\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_744\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sbh5bpapxmlat6m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_745\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_746\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-sbh5bpapxmlat6m","anchor":"1777455"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-gxwzyxuo7h5xyw5\/","ref":"T95253#1781117"},[],{"anchor":"1781117"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_752\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_753\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gxwzyxuo7h5xyw5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_754\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_755\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gxwzyxuo7h5xyw5","anchor":"1781117"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ubci3dlcittpsqa\/","ref":"T95253#1781721"},[],{"anchor":"1781721"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_761\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_762\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ubci3dlcittpsqa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_763\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_764\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ubci3dlcittpsqa","anchor":"1781721"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-2zd26mxwpgw6wjx\/","ref":"T95253#1781831"},[],{"anchor":"1781831"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_770\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_771\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2zd26mxwpgw6wjx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_772\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_773\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2zd26mxwpgw6wjx","anchor":"1781831"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-eubtdrnbkegmcpq\/","ref":"T95253#1781842"},[],{"anchor":"1781842"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_779\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_780\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_215\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eubtdrnbkegmcpq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_781\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_782\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eubtdrnbkegmcpq","anchor":"1781842"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-xljtzesg3c6vufw\/","ref":"T95253#1781844"},[],{"anchor":"1781844"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_217\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_788\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_789\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_219\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xljtzesg3c6vufw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_790\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_791\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xljtzesg3c6vufw","anchor":"1781844"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-3ll2776zeyzyjzd\/","ref":"T95253#1781920"},[],{"anchor":"1781920"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_221\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_797\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_798\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_223\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3ll2776zeyzyjzd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_799\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_800\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3ll2776zeyzyjzd","anchor":"1781920"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ljz7cwr7wdg5qtd\/","ref":"T95253#1781940"},[],{"anchor":"1781940"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_225\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_806\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_807\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_227\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ljz7cwr7wdg5qtd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_808\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_809\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ljz7cwr7wdg5qtd","anchor":"1781940"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-qwivnvllj3isbcr\/","ref":"T95253#1781948"},[],{"anchor":"1781948"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_229\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_815\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_816\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_231\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qwivnvllj3isbcr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_817\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_818\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qwivnvllj3isbcr","anchor":"1781948"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-q4cu5uj6w3zn2ev\/","ref":"T95253#1781977"},[],{"anchor":"1781977"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_233\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_824\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_825\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_235\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-q4cu5uj6w3zn2ev\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_826\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_827\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_237\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-q4cu5uj6w3zn2ev\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_828\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-q4cu5uj6w3zn2ev","anchor":"1781977"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-zvv2aqt7miiaaxj\/","ref":"T95253#1782049"},[],{"anchor":"1782049"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_239\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_834\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_835\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_241\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zvv2aqt7miiaaxj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_836\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_837\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-zvv2aqt7miiaaxj","anchor":"1782049"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ig5dbctetkchtyr\/","ref":"T95253#1782090"},[],{"anchor":"1782090"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_243\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_843\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_844\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_245\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ig5dbctetkchtyr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_845\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_846\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_247\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-ig5dbctetkchtyr\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_847\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ig5dbctetkchtyr","anchor":"1782090"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ro2is6iqq6zfgij\/","ref":"T95253#1782118"},[],{"anchor":"1782118"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_249\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_853\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_854\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_251\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ro2is6iqq6zfgij\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_855\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_856\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ro2is6iqq6zfgij","anchor":"1782118"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-c22zantu63u5jdv\/","ref":"T95253#1782155"},[],{"anchor":"1782155"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_253\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_862\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_863\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_255\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-c22zantu63u5jdv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_864\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_865\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-c22zantu63u5jdv","anchor":"1782155"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-p7nzz6jwrbt2lhp\/","ref":"T95253#1782369"},[],{"anchor":"1782369"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_257\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_871\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_872\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_259\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-p7nzz6jwrbt2lhp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_873\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_874\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-p7nzz6jwrbt2lhp","anchor":"1782369"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-sl4bjvyotca6o6f\/","ref":"T95253#1795557"},[],{"anchor":"1795557"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_261\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_880\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_881\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_263\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sl4bjvyotca6o6f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_882\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_883\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sl4bjvyotca6o6f","anchor":"1795557"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z7jl3mllp3ionjc","anchor":"1798099"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pdry3v6mmjgwch5\/","ref":"T95253#1798940"},[],{"anchor":"1798940"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_265\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_892\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_893\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_267\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pdry3v6mmjgwch5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_894\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_895\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pdry3v6mmjgwch5","anchor":"1798940"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-silo7itujlypclm\/","ref":"T95253#1798994"},[],{"anchor":"1798994"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_269\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_901\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_902\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_271\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-silo7itujlypclm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_903\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_904\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-silo7itujlypclm","anchor":"1798994"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-y6sjydgiblyyzke\/","ref":"T95253#1807247"},[],{"anchor":"1807247"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_273\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_910\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_911\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_275\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-y6sjydgiblyyzke\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_912\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_913\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-y6sjydgiblyyzke","anchor":"1807247"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-v4ayfgws5qhbebz\/","ref":"T95253#1828490"},[],{"anchor":"1828490"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_277\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_919\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_920\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_279\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v4ayfgws5qhbebz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_921\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_922\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-v4ayfgws5qhbebz","anchor":"1828490"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ebn2csxtolg7e43\/","ref":"T95253#1829601"},[],{"anchor":"1829601"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_281\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_928\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_929\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_283\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ebn2csxtolg7e43\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_930\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_931\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_285\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-ebn2csxtolg7e43\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_932\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ebn2csxtolg7e43","anchor":"1829601"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nslhfud5r5l6eh4","anchor":"1832398"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-wimvou7t5vt6o72\/","ref":"T95253#1833191"},[],{"anchor":"1833191"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_287\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_941\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_942\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_289\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wimvou7t5vt6o72\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_943\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_944\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wimvou7t5vt6o72","anchor":"1833191"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}