for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rabo5x3cc3cpt6zd55fh\/PHID-FILE-mrot2z6mektp57wjotpu\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/RobH\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5206570\" id=\"5206570\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_75\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RobH\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eRobH\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5206570\" data-sigil=\"has-tooltip\" data-meta=\"0_74\"\u003e\u003cspan class=\"screen-only\"\u003eMay 22 2019, 8:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-22 20:59:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_76\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RobH\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eRobH\u003c\/a\u003e mentioned this in \u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eUnknown Object (Task)\u003c\/span\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_88\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rabo5x3cc3cpt6zd55fh\/PHID-FILE-mrot2z6mektp57wjotpu\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/RobH\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5206600\" id=\"5206600\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_86\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RobH\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eRobH\u003c\/a\u003e assigned this task to \u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\"\u003eAndrew\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#5206600\" data-sigil=\"has-tooltip\" data-meta=\"0_85\"\u003e\u003cspan class=\"screen-only\"\u003eMay 22 2019, 9:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-22 21:08:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_87\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RobH\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003eRobH\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/952\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/952\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003eRacking Tasks \u003c\/a\u003e on the \u003ca href=\"\/tag\/ops-eqiad\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eops-eqiad\u003c\/a\u003e board.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_83\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Andrew\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Andrew\u003c\/span\u003e\u003c\/a\u003e or \u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e: Since you both were commenting on the hardware specification task, I'm assuming you would also be the ones to ask about the networking requirements\/vlans for these systems as well as the redundancy requirements?\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhat will be the hostnames for these systems?  Update \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Infrastructure_naming_conventions\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Infrastructure_naming_conventions\u003c\/a\u003e if its new.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHow many interfaces need to be network connected and to what vlans?\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIdeally we can locate these outside of row B, since row B 10G is VERY crowded due to cloudvirts and other row B restricted hosts.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCan any of these systems share a rack or are they horizontally redundant between the three and need different racks?\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5206630\" id=\"5206630\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5206630\" data-sigil=\"has-tooltip\" data-meta=\"0_95\"\u003e\u003cspan class=\"screen-only\"\u003eMay 22 2019, 9:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-22 21:26:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_93\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI vote to add cloudosd1xxx to the naming conventions unless my team rebels against that.  The related monitor nodes would end up cloudmon1xxx.  They are more possibly multi-purpose, but they'll be primary monitors for Ceph.  Since this is the PoC, we can always revisit that in the future.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eI'll get back to you soon on the network placement and so forth.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThey will be horizontally redundant between the three and should have different racks.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_106\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5211018\" id=\"5211018\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5211018\" data-sigil=\"has-tooltip\" data-meta=\"0_104\"\u003e\u003cspan class=\"screen-only\"\u003eMay 24 2019, 5:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-05-24 17:38:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_102\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI can say after a bit more research and experimentation that it would be good to be able to split between a data network that connects it with the cloudvirts and friends and a "private" management network that routes traffic for things like rebuilds of the storage devices to prevent such an event from filling the network to the cloudvirts (entirely possible on 10G).  The three monitor hosts will need to be connected the same way, however we do it.  I'll keep doing more homework on the network, but for now, I've got that (however possible or not that is).  \u003ca href=\"\/p\/Andrew\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Andrew\u003c\/span\u003e\u003c\/a\u003e out of curiosity, to get around the row B restriction, this basically looks like putting the Ceph "public" network on the actual public network, right?  Then the backend traffic could be wired to the private network, since that is all intra-cluster. (Ceph calls it the "private" network).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5243547\" id=\"5243547\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5243547\" data-sigil=\"has-tooltip\" data-meta=\"0_113\"\u003e\u003cspan class=\"screen-only\"\u003eJun 7 2019, 6:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-07 18:24:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_111\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_112\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSo for network: if possible, can we do one port on the public lan and one port on the private?  \u003ca href=\"\/p\/RobH\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@RobH\u003c\/span\u003e\u003c\/a\u003e \u003cbr \/\u003e\nEverything else in my first comment should be right.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_124\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5243552\" id=\"5243552\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5243552\" data-sigil=\"has-tooltip\" data-meta=\"0_122\"\u003e\u003cspan class=\"screen-only\"\u003eJun 7 2019, 6:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-07 18:26:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_120\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_121\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNote: the hosts aren't expected to work as routers.  They just should have management traffic separated if we can properly run them through the paces.  We can probably make things work without that if we have to.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rabo5x3cc3cpt6zd55fh\/PHID-FILE-mrot2z6mektp57wjotpu\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/RobH\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5243578\" id=\"5243578\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RobH\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eRobH\u003c\/a\u003e reassigned this task from \u003ca href=\"\/p\/Andrew\/\" class=\"phui-handle phui-link-person\"\u003eAndrew\u003c\/a\u003e to \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\"\u003eayounsi\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5243578\" data-sigil=\"has-tooltip\" data-meta=\"0_131\"\u003e\u003cspan class=\"screen-only\"\u003eJun 7 2019, 6:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-07 18:33:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_133\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RobH\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eRobH\u003c\/a\u003e added subscribers: \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003eayounsi\u003c\/a\u003e, \u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003eMoritzMuehlenhoff\u003c\/a\u003e.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_129\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOk, I've synced up with \u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e via IRC, and we have the following questions to be addressed by our network admin(s) to ensure we aren't breaking any rules:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThese hosts need to have a public interface for users to hit, so the primary interface needs to be in the public1-<row letter>-eqiad vlan\/subnet.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe hosts also need to push private CEPH traffic between nodes, and will need their secondary interfaces connected to the private1-<row letter>-eqiad vlan\/subnet.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe above would allow these 3 hosts to be distributed into rows A\/C\/D and allow for greatest horizontal redundancy.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI'm not sure if we have any other hosts bridging the private1\/public1 subnets within a single host, so I'd like to check with \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e or \u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MoritzMuehlenhoff\u003c\/span\u003e\u003c\/a\u003e to ensure this isn't a concern.\u003c\/p\u003e\n\n\u003cp\u003eIf the above isn't a concern, then the racking plan will be:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eone host in rows A\/C\/D\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eprimary 10G on public1 subnet\/vlan\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003esecondary 10G on private1 subnet\/vlan\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rc73mchiefshlqrgquuy\/PHID-FILE-gl42jjecpmjnqcd5dbz2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ayounsi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5264327\" id=\"5264327\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eayounsi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5264327\" data-sigil=\"has-tooltip\" data-meta=\"0_141\"\u003e\u003cspan class=\"screen-only\"\u003eJun 18 2019, 5:55 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-18 05:55:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_139\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_52\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThere is no security or technical limitation on having 1 interface in each vlan as long as they don't go across rows. Hosts in one vlan can already reach the hosts in the other.\u003cbr \/\u003e\nSo the question is more about using extra 10G switch ports, increasing configuration complexity, and load (for the cluster in general).\u003c\/p\u003e\n\n\u003cp\u003eI went through \u003ca href=\"http:\/\/docs.ceph.com\/docs\/mimic\/rados\/configuration\/network-config-ref\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttp:\/\/docs.ceph.com\/docs\/mimic\/rados\/configuration\/network-config-ref\/\u003c\/a\u003e\u003cbr \/\u003e\nThe two reasons stated for that separation are:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePerformances (replication traffic impacting client traffic)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSecurity (client DoSing the one interface, disrupting health-check like traffic)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSome questions about the above and overall design\/flows:\u003cbr \/\u003e\n\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e \u003cbr \/\u003e\nIs there more doc on the network aspect of CEPH I should read?\u003cbr \/\u003e\nHow much traffic are you expecting on both networks? (average\/peak) As well as cross row?\u003cbr \/\u003e\nIs there a risk of saturating the 40G links between racks\/rows?\u003cbr \/\u003e\nHow much performance differences is expected from using 2 interfaces instead of 1?\u003cbr \/\u003e\nWho will be the clients? Cloud only or the public internet as well?\u003cbr \/\u003e\nIs there a risk\/concern that the clients DoS the cluster?\u003cbr \/\u003e\nCan user or replication traffic be rate-limited?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/vlre5c7v4hsydu5xy2ad\/PHID-FILE-dboe6joqdcqjrpk3d7hn\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Cmjohnson\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5269123\" id=\"5269123\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_151\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Cmjohnson\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Cmjohnson\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-3vapkff6dyfqaor\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5269123\" data-sigil=\"has-tooltip\" data-meta=\"0_150\"\u003e\u003cspan class=\"screen-only\"\u003eJun 19 2019, 4:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-19 16:56:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Cmjohnson\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Cmjohnson\u003c\/a\u003e subscribed.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_148\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_54\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e  I will need very clear instructions on which racks\/rows these servers can go in before I physically rack and cable.   Once that is figured out please update the task.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5271528\" id=\"5271528\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5271528\" data-sigil=\"has-tooltip\" data-meta=\"0_160\"\u003e\u003cspan class=\"screen-only\"\u003eJun 20 2019, 3:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-20 15:37:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_158\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e Ceph docs are vague at best or tend to ask you to read dissertations eventually.  Overall, everything comes back to "test it in your cluster and see".  Ceph is capable of saturating 10G links under heavy load (and the private link would be able to saturate during node failures for rebuilds).  A 40G link would be harder to saturate, but it is theoretically possible.  This is a PoC, so my intent is to break it every which way and put it under test loads.  We would certainly want to keep an eye on those links during tests (are you able to point me to where I could do that?).\u003c\/p\u003e\n\n\u003cp\u003e" How much traffic are you expecting on both networks? (average\/peak) As well as cross row?" - basically the answer is "What a good question!"  We should learn more from this test.  This is meant to help define the design of the build-out of a fully operational cluster, so monitoring what it does is important.  This will be a fairly low-latency cluster if we are able to design it well, but it will be limited by its 10G link on the public side.  The private side traffic will be dictated by traffic on the public and and traffic for rebuilds and reshuffles of data.  I expect that side to be smaller but significant, except all that traffic will be between ceph nodes, not with clients.\u003c\/p\u003e\n\n\u003cp\u003eI can say that there is a very significant performance difference if using 1 interface instead of 2 because Ceph does around as much chatter among itself as it does with clients (unlike NFS).  So both links will be as chatty as the clients make them.\u003c\/p\u003e\n\n\u003cp\u003eThe clients will be cloud only, but the public IP is being used to avoid firewall\/vlan issues for that connection.  It will be providing block devices to the cloudvirts, VMs and likely an NFS replacement service as well.  This cluster itself is merely a test\/PoC that will remain in production after build-out.  It will be closer to 10 nodes in a final form (we think...based on this testing that will happen after we build this test cluster).  That is imporant to consider for distributing load in the build-out phase and perhaps that will help inform your thoughts now as well.  That's a lot of 10G links, and it is also mostly aimed at cross-row traffic because it will be talking to cloud and ceph.\u003c\/p\u003e\n\n\u003cp\u003eDoS: I am concerned about DoS from the public side because we are doing the main traffic on that end, but Ceph is authenticated and will be firewalled, so hopefully, that will be mitigated entirely by that.  For the most part, if we trust ferm\/iptables, we should be safe there except from Cloud clients.  We aren't too worried Cloud clients will do that.\u003c\/p\u003e\n\n\u003cp\u003eRate-limiting traffic is likely to collapse the cluster.  We can experiment with it mostly on the public side during the PoC.  On the private side, it is more likely to cause problems.  This is meant to operate like a SAN in some ways for the clients, though, so it would cause problems if rate-limited.  We'll be testing whatever knobs we have to keep some quality of service available.\u003c\/p\u003e\n\n\u003cp\u003eI will add that plenty of people build new networks just for Ceph (partly to get jumbo frames).  Our loads are somewhat unpredictable, but I'm hoping to huddle with you a bit during the PoC so we can talk about how our tests and loads affect things.\u003c\/p\u003e\n\n\u003cp\u003eDoes that help?  I didn't answer in a very orderly fashion, but I tried to cover everything I know so far.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_171\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/mdcg6p3p2o5viflgmprf\/PHID-FILE-4ruc2q2pqoq5orhdnh43\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/faidon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273412\" id=\"5273412\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_170\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/faidon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003efaidon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273412\" data-sigil=\"has-tooltip\" data-meta=\"0_169\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 9:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 09:37:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_167\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_59\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T224188#5271528\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT224188#5271528\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eCeph is capable of saturating 10G links under heavy load\u003cbr \/\u003e\n[...]\u003cbr \/\u003e\nRate-limiting traffic is likely to collapse the cluster.\u003cbr \/\u003e\n[...]\u003cbr \/\u003e\nI will add that plenty of people build new networks just for Ceph (partly to get jumbo frames).\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eSo, what's problematic here is that both our switch leaf-spine uplinks, as well as our spine-router uplinks (used for cross-row traffic, say from row A to row B) are 40G. With multiple nodes with 10G links, and given the existing link utilization (peaking at 8G on 5m avg), this means that about ~3 of them chattering at full speed (either with each other for replication, and\/or with their -now 10G- clients) would be enough to start causing datacenter-wide congestion issues for all services, and thus outage-type scenarios (think LVS-to-varnish, appserver-to-database, etc. etc.)\u003c\/p\u003e\n\n\u003cp\u003eSo... that would obviously be a major concern :) It sounds like the intention is to load test with 3-6 10G ports across multiple racks\/rows which... we cannot accomodate with our existing network setup without causing widespread issues :(\u003c\/p\u003e\n\n\u003cp\u003eFor both the PoC and the production setup, we need either guarantees for a (relatively small) amount of traffic this will carry (possibly with rate limits? 1G ports? or limiting in specific rack or rows?), or to rethink the network setup for this. Example options being to isolate this into an entirely new network, and\/or redo our data center design -- both of which being major and costly endeavours.\u003c\/p\u003e\n\n\u003cp\u003e(Also, note that no resourcing asks for networking for FY19-20 made the cuts, and a network upgrade from, say, 40G to 100G, requires new router linecards and new spines, and would have a cost in the ballpark of $500-600k and be a 3-4 quarter project, and that's a conservative estimate\u2026)\u003c\/p\u003e\n\n\u003cp\u003eI don't have bright ideas or easy ways out at this point, and honestly I haven't really thought about all this much. But, this clearly surfaces as an important risk at this point, as it raises "can impact site stability" concerns. These apply for sure to the proposed production setup, but even for the PoC itself given the intention is to put it under (heavy?) test loads. So, apologies for the extra scrutiny and potential setbacks, but I think it's warranted :\/\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5277075\" id=\"5277075\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5277075\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 3:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 03:16:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_176\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e100% agree with you \u003ca href=\"\/p\/faidon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@faidon\u003c\/span\u003e\u003c\/a\u003e, and I appreciate the reply.  I'm aiming to avoid any sugar-coating in my assessments of risks until I have more data (especially with 40G uplinks that are widely shared), partly to open conversations and make sure we design carefully.  After a bit of time to think about this, I have some more thoughts.\u003c\/p\u003e\n\n\u003cp\u003eI should soften some of my statements a little with the idea that any test that gets close to filling a 10G link would be stopped and noted.  The goal is to identify where that could happen and resolve the risk (or plan rollout\/scaling accordingly).  The network design is certainly a limitation on how much we can scale ceph and how we can place nodes (and I'm learning more about our network design now, which is great!).  The general fix for such saturation issues is to add more nodes, but those nodes need to be spread in such a way that they don't endanger uplinks.  This is also why fewer, larger nodes can be a problem for the backend\/private network because a very large OSD node taking an outage will cause a LOT of backend chatter to rebuild, on a scale that is entirely determined by the size of the node's storage and the speed of the network it has available.  Our hope is to understand any conditions that can reach that level and work against them in our setup.\u003c\/p\u003e\n\n\u003cp\u003eI also think that depending on how we limit the rate of traffic, it could be made workable so that it will not effectively "collapse" the cluster (simply like using a different size link--1G links are not advised for all SSD, obviously, when used at high speed).  By collapse, I mean made unstable in a really bad way where things become unusable to clients and the chatter on the backend gets far worse. If we are rate-limiting in the right way (and it calls for experimentation) we should be able to simply extend rebuild times for the back-end network (not just for the public side), but the effects very well could be unexpected on the stability of the cluster--and we would know for certain until we try it with a sensible load because all the blasted docs say "it depends".\u003cbr \/\u003e\nI was initially thinking of rate-limiting mostly in terms of how we do it on the NFS hosts (via the servers), and that's really not likely to work at first glance.\u003cbr \/\u003e\nIt would be really great to discuss what options are available for rate limiting both sides and to test them.  If none are available on the network end, I can go digging on the host side (but I'm less positive about that side so far).\u003c\/p\u003e\n\n\u003cp\u003eThis has me thinking a lot about scaling strategies and so forth as well, but I'll save that for another discussion--ditto for the fact that our network interface saturation monitoring is still diamond-based :-p\u003c\/p\u003e\n\n\u003cp\u003eOnce we have a running cluster to beat on and actually see what simulations of our loads actually act like (accepting that our technical contributors are rather creative in their ability to make new loads), we may find that the ceph project might be more limited than we'd prefer.  We might also find out that its pretty easy to rate limit and monitor it for safety (which would be really nice).  We might even find out that without jumbo frames, our performance is really weak, and it needs to be for certain limited purposes only anyway.\u003c\/p\u003e\n\n\u003cp\u003eSo, if you ask me "is there a risk", I'll definitely say, "yes" because I cannot prove otherwise yet, and people out there in the wild have seen these things happen.\u003cbr \/\u003e\nDo I believe we will definitely do that much damage?  On the PoC, I would be very unpleasantly surprised if we did, and we will plan to avoid it carefully (including making monitoring the interfaces a hard requirement, following this conversation).  On full rollout: I hope to have a better idea of the risk level after the PoC and also hope to have strategies in place to address more possible failure modes including risks to production systems.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e Is there a way my team could monitor any potential impact on uplinks during tests (besides the obvious math and drawing pictures of the placement within the network)?  If not, I'll want to draw myself a network map of them and follow the way the links are interacting.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5277076\" id=\"5277076\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_188\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Bstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5277076\" data-sigil=\"has-tooltip\" data-meta=\"0_187\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 3:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 03:37:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_185\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNote: there are rate limits that can be set within openstack for this as well...but in some versions, they don't work right at all (they get ignored in some cases \u003ca href=\"https:\/\/bugzilla.redhat.com\/show_bug.cgi?id=1476830\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/bugzilla.redhat.com\/show_bug.cgi?id=1476830\u003c\/a\u003e), but this is also things we want to be testing.  That won't help back-end stuff, etc either.  It's just a note.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rc73mchiefshlqrgquuy\/PHID-FILE-gl42jjecpmjnqcd5dbz2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ayounsi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5277181\" id=\"5277181\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eayounsi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5277181\" data-sigil=\"has-tooltip\" data-meta=\"0_196\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 6:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 06:12:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_194\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eour network interface saturation monitoring is still diamond-based\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWhat does that mean?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIs there a way my team could monitor any potential impact on uplinks during tests\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThe easiest way is to look at LibreNMS, on both cr1 and \u003ca href=\"https:\/\/librenms.wikimedia.org\/device\/device=2\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ecr2\u003c\/a\u003e, ae1 goes to row A, ae2 to row B, ae3 to row C, ae4 to row D.\u003cbr \/\u003e\nNote that LibreNMS have a 5min granularity. That mean if a sudden spike of traffic appear, it will not get noticed right away.\u003cbr \/\u003e\nWe also have alerting for when the link reach 80% utilization, with the same 5min caveat.\u003cbr \/\u003e\nA "real time" view exists (eg. \u003ca href=\"https:\/\/librenms.wikimedia.org\/device\/device=2\/tab=port\/port=139\/view=realtime\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/librenms.wikimedia.org\/device\/device=2\/tab=port\/port=139\/view=realtime\/\u003c\/a\u003e ) but it needs to be used carefully to not overwhelm the router's SNMP daemon.\u003c\/p\u003e\n\n\u003cp\u003eJumbo frames are enabled everywhere on the switch side, so make sure the proper MTU is set on the host side if you want to use it as a "natural" rate limiter.\u003c\/p\u003e\n\n\u003cp\u003eRate limiting on the network side is usually not advised as it have a bad performance hit. Better send the packets slower than create an artificial bottleneck that the sending host have to detect and work around (TCP scaling, etc...).\u003c\/p\u003e\n\n\u003cp\u003eOnly using the public interface could be a way to "naturally" rate limit the cluster (10G total, instead of a theoretical max of 20G per host).\u003c\/p\u003e\n\n\u003cp\u003eOther option I see would be to keep all the nodes in row B, keeping the impact radius of the cluster miss-behaving to that one row. This also removes the cross row client traffic.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_207\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dehmh3scjpmoumxznua7\/PHID-FILE-4esn6uidadgmq7ck5yod\/alphanumeric_aleo-white_M.png-_b38ba9-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MoritzMuehlenhoff\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5277307\" id=\"5277307\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_206\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003eMoritzMuehlenhoff\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5277307\" data-sigil=\"has-tooltip\" data-meta=\"0_205\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 7:24 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 07:24:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_203\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_204\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T224188#5277181\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT224188#5277181\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eour network interface saturation monitoring is still diamond-based\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWhat does that mean?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eDiamond is our old tool to collect local system metrics. It has been superseded by Prometheus mostly, but there are still some remaining use cases in WMCS which aren't migrated yet: \u003cbr \/\u003e\n\u003ca href=\"https:\/\/phabricator.wikimedia.org\/T210993\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_19\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T210993\u003c\/span\u003e\u003c\/a\u003e\u003cbr \/\u003e\n\u003ca href=\"https:\/\/phabricator.wikimedia.org\/T210850\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_21\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T210850\u003c\/span\u003e\u003c\/a\u003e\u003cbr \/\u003e\n\u003ca href=\"https:\/\/phabricator.wikimedia.org\/T210991\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_23\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T210991\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_216\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5278005\" id=\"5278005\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_215\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5278005\" data-sigil=\"has-tooltip\" data-meta=\"0_214\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 11:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 11:14:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_212\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_213\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWorth noting that even though we will be using 10G links, we don't expect them to be fully used \u003cstrong\u003ein any case\u003c\/strong\u003e in the short term.\u003c\/p\u003e\n\n\u003cp\u003eIf we set all the CEPH servers in row B (for the initial PoC) we could avoid saturating any upstream link or device. Only Top-of-rack switches (asw2-b-eqiad).\u003cbr \/\u003e\nI know 10G in row B is limited right now, but I don't see any other option.\u003cbr \/\u003e\nRate limiting should be possible both at client and server levels (iptables should do the trick, or alternatively, a similar tc setup like we have right now with NFS). Also possible at network hardware level (i.e, switches) I think. Also, what \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e mentioned could be a good starting point, using only 1x10G in each servers instead of 2x10G.\u003c\/p\u003e\n\n\u003cp\u003eSo here is my proposal:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003erack each of the 6 new servers (3x mons, 3x OSDs) in different row B racks, using 1x10G links in each server\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eimplement any simple rate limiting in case we want to be extra sure that we don't fully use 10G in every server\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI'm just trying to find a compromise between our goals and what the network can handle.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_225\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rc73mchiefshlqrgquuy\/PHID-FILE-gl42jjecpmjnqcd5dbz2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ayounsi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5278081\" id=\"5278081\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_224\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003eayounsi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5278081\" data-sigil=\"has-tooltip\" data-meta=\"0_223\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 11:53 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 11:53:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_221\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_222\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat sounds reasonable for the PoC, depending on rack space. \u003ca href=\"\/p\/faidon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@faidon\u003c\/span\u003e\u003c\/a\u003e for the last word.\u003c\/p\u003e\n\n\u003cp\u003eNote that we don't have visibility in the cross virtual chassis links. Adding it to LibreNMS is possible but would require dev time.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_234\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5278125\" id=\"5278125\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5278125\" data-sigil=\"has-tooltip\" data-meta=\"0_232\"\u003e\u003cspan class=\"screen-only\"\u003eJun 24 2019, 12:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-24 12:14:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_230\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T224188#5278081\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT224188#5278081\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThat sounds reasonable for the PoC, depending on rack space. \u003ca href=\"\/p\/faidon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@faidon\u003c\/span\u003e\u003c\/a\u003e for the last word.\u003c\/p\u003e\n\n\u003cp\u003eNote that we don't have visibility in the cross virtual chassis links. Adding it to LibreNMS is possible but would require dev time.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eFair. The virtual chassis links, Are they DAC cables or fiber? In any case, how much throughput do they have?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-22bsa5u75jz3ci3wnplu","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ktuojnvco4fpzmyhzyaf","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-ovgpv2sbjrjywp3yvcbi"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-liabgr64xtmbq7bhq4r6"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-my6tgagcjpmg43ydinof"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ktuojnvco4fpzmyhzyaf","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ktuojnvco4fpzmyhzyaf","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-5dinlvlxad5q3rtcx546"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-dibwuxnegzngogp5bh7i"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-kcxcherrayes27tfxxpj"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-i6tvvqvwbaiubtfcvsn2"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-wr7jlgkb5qk5ykssqwn7"}},{"phid":"PHID-XACT-TASK-ttbtvdit5oczxqw"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-ny3rueqojppouqt"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-xqgqhnb2xqqush3"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-cs4bm66mi2gqeu3"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-wzt4lbvseijr2yr"},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xy6c3ul27f336aaedx2d"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"phid":"PHID-XACT-TASK-6ndmdor4l5yznun"},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5"}},{"phid":"PHID-XACT-TASK-5gtv2yhplra45po"},{"hovercardSpec":{"objectPHID":"PHID-USER-idbur34nizbyyqlnwaqc"}},{"phid":"PHID-XACT-TASK-r7a6xxotzguxnop"},{"hovercardSpec":{"objectPHID":"PHID-USER-idbur34nizbyyqlnwaqc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-f35uf6eiyga65iz"},{"hovercardSpec":{"objectPHID":"PHID-USER-ktuojnvco4fpzmyhzyaf"}},{"phid":"PHID-XACT-TASK-yu4kapf2ijzh7zm"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-xadcmmrgdulpju7"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-snqntiuci4lkc4f"},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5"}},{"phid":"PHID-XACT-TASK-uqt4ma4fvonmfu5"},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"phid":"PHID-XACT-TASK-c6hl6kbc6e2r7hx"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-j3q4dtaqtsfxkxn"},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5"}},{"phid":"PHID-XACT-TASK-4qkz2oleudlpe4z"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-mp4rgkxv3bkw7ty"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-kga5wu46gwguk4f","anchor":"5206570"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ttbtvdit5oczxqw\/","ref":"T224188#5206600"},[],{"anchor":"5206600"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_78\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ttbtvdit5oczxqw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_80\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-ttbtvdit5oczxqw\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_82\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-rytuiycgchowjpj","anchor":"5206600"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ny3rueqojppouqt\/","ref":"T224188#5206630"},[],{"anchor":"5206630"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_89\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ny3rueqojppouqt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_91\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_92\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ny3rueqojppouqt","anchor":"5206630"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xqgqhnb2xqqush3\/","ref":"T224188#5211018"},[],{"anchor":"5211018"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_98\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_99\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xqgqhnb2xqqush3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_100\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xqgqhnb2xqqush3","anchor":"5211018"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cs4bm66mi2gqeu3\/","ref":"T224188#5243547"},[],{"anchor":"5243547"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_107\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_108\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cs4bm66mi2gqeu3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_109\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_110\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cs4bm66mi2gqeu3","anchor":"5243547"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wzt4lbvseijr2yr\/","ref":"T224188#5243552"},[],{"anchor":"5243552"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_116\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_117\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wzt4lbvseijr2yr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_118\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_119\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wzt4lbvseijr2yr","anchor":"5243552"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6ndmdor4l5yznun\/","ref":"T224188#5243578"},[],{"anchor":"5243578"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_125\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_126\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6ndmdor4l5yznun\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_127\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_128\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-m3ltpbqpzu6i7ql","anchor":"5243578"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5gtv2yhplra45po\/","ref":"T224188#5264327"},[],{"anchor":"5264327"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_135\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5gtv2yhplra45po\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5gtv2yhplra45po","anchor":"5264327"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-r7a6xxotzguxnop\/","ref":"T224188#5269123"},[],{"anchor":"5269123"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r7a6xxotzguxnop\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_146\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-3vapkff6dyfqaor","anchor":"5269123"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-f35uf6eiyga65iz\/","ref":"T224188#5271528"},[],{"anchor":"5271528"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_154\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-f35uf6eiyga65iz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_156\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_157\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-f35uf6eiyga65iz","anchor":"5271528"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yu4kapf2ijzh7zm\/","ref":"T224188#5273412"},[],{"anchor":"5273412"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_163\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yu4kapf2ijzh7zm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_165\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yu4kapf2ijzh7zm","anchor":"5273412"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xadcmmrgdulpju7\/","ref":"T224188#5277075"},[],{"anchor":"5277075"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xadcmmrgdulpju7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_174\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xadcmmrgdulpju7","anchor":"5277075"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-snqntiuci4lkc4f\/","ref":"T224188#5277076"},[],{"anchor":"5277076"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_181\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-snqntiuci4lkc4f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_183\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-snqntiuci4lkc4f","anchor":"5277076"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uqt4ma4fvonmfu5\/","ref":"T224188#5277181"},[],{"anchor":"5277181"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_190\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_191\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uqt4ma4fvonmfu5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_192\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_193\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uqt4ma4fvonmfu5","anchor":"5277181"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-c6hl6kbc6e2r7hx\/","ref":"T224188#5277307"},[],{"anchor":"5277307"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_199\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_200\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-c6hl6kbc6e2r7hx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_201\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_202\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-c6hl6kbc6e2r7hx","anchor":"5277307"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-j3q4dtaqtsfxkxn\/","ref":"T224188#5278005"},[],{"anchor":"5278005"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_208\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j3q4dtaqtsfxkxn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_210\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_211\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-j3q4dtaqtsfxkxn","anchor":"5278005"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4qkz2oleudlpe4z\/","ref":"T224188#5278081"},[],{"anchor":"5278081"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_217\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4qkz2oleudlpe4z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_219\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4qkz2oleudlpe4z","anchor":"5278081"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mp4rgkxv3bkw7ty\/","ref":"T224188#5278125"},[],{"anchor":"5278125"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mp4rgkxv3bkw7ty\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_228\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mp4rgkxv3bkw7ty","anchor":"5278125"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/45b62efe\/core.pkg.css"]}