for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_66\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7110992\" id=\"7110992\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_65\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003ehashar\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7110992\" data-sigil=\"has-tooltip\" data-meta=\"0_64\"\u003e\u003cspan class=\"screen-only\"\u003eMay 25 2021, 12:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-25 12:11:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7111001\" id=\"7111001\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_68\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_17\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/4273281\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7111001\" data-sigil=\"has-tooltip\" data-meta=\"0_67\"\u003e\u003cspan class=\"screen-only\"\u003eMay 25 2021, 12:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-25 12:11:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/18\/\" data-sigil=\"has-tooltip\" data-meta=\"0_80\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-star\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/14\/\" data-sigil=\"has-tooltip\" data-meta=\"0_82\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-empire\" data-meta=\"0_83\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7111022\" id=\"7111022\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_77\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003eMarostegui\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eMedium\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7111022\" data-sigil=\"has-tooltip\" data-meta=\"0_76\"\u003e\u003cspan class=\"screen-only\"\u003eMay 25 2021, 12:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-25 12:29:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_78\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003eMarostegui\u003c\/a\u003e edited projects, added \u003ca href=\"\/tag\/serviceops\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003eserviceops\u003c\/a\u003e, \u003ca href=\"\/tag\/netops\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003enetops\u003c\/a\u003e; removed \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003eSRE\u003c\/a\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eMarostegui\u003c\/a\u003e added subscribers: \u003ca href=\"\/p\/Legoktm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003eLegoktm\u003c\/a\u003e, \u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eMarostegui\u003c\/a\u003e.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_74\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_75\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eTagging also \u003ca href=\"\/tag\/netops\/\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-briefcase\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003enetops\u003c\/span\u003e\u003c\/a\u003e in case they can help out\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rc73mchiefshlqrgquuy\/PHID-FILE-gl42jjecpmjnqcd5dbz2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ayounsi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7111164\" id=\"7111164\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_92\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eayounsi\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eayounsi\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7111164\" data-sigil=\"has-tooltip\" data-meta=\"0_91\"\u003e\u003cspan class=\"screen-only\"\u003eMay 25 2021, 1:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-25 13:38:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_89\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_28\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI'd defer to DCops\u003cbr \/\u003e\nmgmt is connected to un-managed switches, so we don't have much visibility on this side.\u003cbr \/\u003e\nEither the server's mgmt port is faulty, or the software, or the cable or the switch port.\u003cbr \/\u003e\nI'd recommend to try to change\/fix each of them until it's fully fixed, starting with the low hanging ones.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_96\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/x6fn4twohsiictamz65s\/PHID-FILE-2itdywof6ttpeb6suxa2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Marostegui\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7111183\" id=\"7111183\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_95\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Marostegui\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eMarostegui\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/dc-ops\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eDC-Ops\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7111183\" data-sigil=\"has-tooltip\" data-meta=\"0_94\"\u003e\u003cspan class=\"screen-only\"\u003eMay 25 2021, 1:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-25 13:41:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2gffy5bww6ck2ynuxnpz\/PHID-FILE-jyvkjrrdy7c4mbtu7gb3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/RhinosF1\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7111446\" id=\"7111446\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_98\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/RhinosF1\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003eRhinosF1\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/RhinosF1\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eRhinosF1\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7111446\" data-sigil=\"has-tooltip\" data-meta=\"0_97\"\u003e\u003cspan class=\"screen-only\"\u003eMay 25 2021, 2:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-05-25 14:49:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_112\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_108\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_110\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7154540\" id=\"7154540\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7154540\" data-sigil=\"has-tooltip\" data-meta=\"0_106\"\u003e\u003cspan class=\"screen-only\"\u003eJun 14 2021, 9:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-06-14 09:22:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_104\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_36\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eLooking at the alarm history since midnight UTC at \u003ca href=\"https:\/\/icinga.wikimedia.org\/cgi-bin\/icinga\/history.cgi?host=all\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/icinga.wikimedia.org\/cgi-bin\/icinga\/history.cgi?host=all\u003c\/a\u003e , I have filtered out alarms matching 2\\d\\d\\d.mgmt. There are a few hosts:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003econtint2001.mgmt\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003egerrit2001.mgmt\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003elogstash2021.mgmt\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003elogstash2022.mgmt\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ewdqs2001.mgmt\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eThough they are in different rows \/ racks in the datacenter.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5j7azqdhxkhiqs4edzna\/PHID-FILE-53pcqqlc2km4s76ndxfp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7158317\" id=\"7158317\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eDzahn\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eDzahn\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7158317\" data-sigil=\"has-tooltip\" data-meta=\"0_119\"\u003e\u003cspan class=\"screen-only\"\u003eJun 15 2021, 5:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-06-15 17:30:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_117\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI can confirm since a while these have been happening. The pattern is always:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eonly mgmt\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eonly codfw\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003erandom host\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eresolves itself shortly after\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThat makes me think it should be the management switch.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c5mll4i2ivmjlggdpizk\/PHID-FILE-x2xqxc36bpfpfub7m6rd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Aklapper\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7167660\" id=\"7167660\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_124\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eAklapper\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/infrastructure-foundations\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003eInfrastructure-Foundations\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7167660\" data-sigil=\"has-tooltip\" data-meta=\"0_122\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2021, 8:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-06-21 20:59:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-bell-slash white phui-icon-emblem phui-icon-emblem-red\" data-sigil=\"has-tooltip\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_128\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dehmh3scjpmoumxznua7\/PHID-FILE-4esn6uidadgmq7ck5yod\/alphanumeric_aleo-white_M.png-_b38ba9-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MoritzMuehlenhoff\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7204780\" id=\"7204780\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eMoritzMuehlenhoff\u003c\/a\u003e mentioned this in \u003ca href=\"\/T286274\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003eT286274: mgmt on logstash2021 inaccessible\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7204780\" data-sigil=\"has-tooltip\" data-meta=\"0_126\"\u003e\u003cspan class=\"screen-only\"\u003eJul 12 2021, 8:47 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-07-12 08:47:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5j7azqdhxkhiqs4edzna\/PHID-FILE-53pcqqlc2km4s76ndxfp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7228005\" id=\"7228005\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7228005\" data-sigil=\"has-tooltip\" data-meta=\"0_135\"\u003e\u003cspan class=\"screen-only\"\u003eJul 21 2021, 4:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-07-21 16:43:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_133\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eACKed some more today, gerrit2001.mgmt,  wdqs2002.mgmt\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_146\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7348063\" id=\"7348063\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7348063\" data-sigil=\"has-tooltip\" data-meta=\"0_144\"\u003e\u003cspan class=\"screen-only\"\u003eSep 13 2021, 12:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-09-13 12:04:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_142\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_47\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat is still happening from time to time. Any person or team I can raise this too?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_159\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5j7azqdhxkhiqs4edzna\/PHID-FILE-53pcqqlc2km4s76ndxfp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Dzahn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7385095\" id=\"7385095\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Dzahn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003eDzahn\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7385095\" data-sigil=\"has-tooltip\" data-meta=\"0_157\"\u003e\u003cspan class=\"screen-only\"\u003eSep 28 2021, 6:18 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-09-28 18:18:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_155\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@hashar\u003c\/span\u003e\u003c\/a\u003e This should be between netops and dcops I think.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_172\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_168\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_169\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_170\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7394226\" id=\"7394226\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_167\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003ehashar\u003c\/a\u003e edited projects, added \u003ca href=\"\/tag\/ops-codfw\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003eops-codfw\u003c\/a\u003e; removed \u003ca href=\"\/tag\/infrastructure-foundations\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003eInfrastructure-Foundations\u003c\/a\u003e, \u003ca href=\"\/tag\/netops\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003enetops\u003c\/a\u003e, \u003ca href=\"\/tag\/serviceops\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eserviceops\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7394226\" data-sigil=\"has-tooltip\" data-meta=\"0_166\"\u003e\u003cspan class=\"screen-only\"\u003eOct 1 2021, 9:31 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-10-01 09:31:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_164\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_165\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eClearing a few projects.  \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e mentioned at \u003ca href=\"\/T283582#7111164\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT283582#7111164\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e that it is most probably an issue with an unmanaged switch for the management network.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_183\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xh2ph2yp26zepxxurglq\/PHID-FILE-w7ofzuyouv2jmz2zuzuo\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/cmooney\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7394322\" id=\"7394322\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_180\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003ecmooney\u003c\/a\u003e claimed this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7394322\" data-sigil=\"has-tooltip\" data-meta=\"0_179\"\u003e\u003cspan class=\"screen-only\"\u003eOct 1 2021, 10:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-10-01 10:40:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_181\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003ecmooney\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/netops\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003enetops\u003c\/a\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003ecmooney\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/cmooney\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003ecmooney\u003c\/a\u003e.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_177\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_178\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_59\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks \u003ca href=\"\/p\/hashar\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@hashar\u003c\/span\u003e\u003c\/a\u003e.  I would agree with \u003ca href=\"\/p\/ayounsi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ayounsi\u003c\/span\u003e\u003c\/a\u003e's analysis, if considering contint2001.mgmt \u003cstrong\u003ein isolation\u003c\/strong\u003e.\u003c\/p\u003e\n\n\u003cp\u003eBut given your subsequent reply and Daniel's response confirming it is spread across racks\/rows, but confined to codfw, I think it's unlikely to be the result of issues on the unmanaged management switches.  It's unlikely we'd have multiple of them faulty, and restricted only to codfw.  It's not \u003cem\u003eimpossible\u003c\/em\u003e that we purchased a dodgy batch of Cat5 cables which have been used there, I did experience an issue like that before, but it's extremely unlikely.\u003c\/p\u003e\n\n\u003cp\u003eSo, given the codfw only pattern, I think it's more likely to be something with the \u003ctt class=\"remarkup-monospaced\"\u003emr1-codfw\u003c\/tt\u003e firewall, or \u003ctt class=\"remarkup-monospaced\"\u003emsw1-codfw\u003c\/tt\u003e aggregation switch (that connects all the unmanaged ones).  I've checked in LibreNMS and the associated links, CPU checks and other relevant metrics look good for both of these devices, so nothing is jumping out at me.  But I can't say for sure there is no problem at this layer.\u003c\/p\u003e\n\n\u003cp\u003eI observe some packet loss when sending high-speed pings from mr1-codfw to any device, for instance pinging the CRs over direct link.  However when sending such pings to devices connected via management switches the results are much better towards the CR mgmt interfaces than to server mgmt interfaces:\u003c\/p\u003e\n\n\u003cp\u003eCR via uplink:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ecmooney@mr1-codfw> ping 208.80.153.206 size 1400 do-not-fragment rapid count 1000 \nPING 208.80.153.206 (208.80.153.206): 1400 data bytes\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n--- 208.80.153.206 ping statistics ---\n1000 packets transmitted, 993 packets received, 0% packet loss\nround-trip min\/avg\/max\/stddev = 1.277\/2.028\/57.115\/2.675 ms\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eCR via msw1-codfw:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ecmooney@mr1-codfw> ping 10.193.0.12 size 1400 do-not-fragment rapid count 1000     \nPING 10.193.0.12 (10.193.0.12): 1400 data bytes\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!.!!!!.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n--- 10.193.0.12 ping statistics ---\n1000 packets transmitted, 997 packets received, 0% packet loss\nround-trip min\/avg\/max\/stddev = 1.181\/3.004\/683.287\/22.545 ms\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003econtint2001.mgmt via msw1-codfw:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003ecmooney@mr1-codfw> ping 10.193.2.250 size 1400 do-not-fragment rapid count 1000    \nPING 10.193.2.250 (10.193.2.250): 1400 data bytes\n!!!!!!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!..!..!..!..!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!..!.!.\n--- 10.193.2.250 ping statistics ---\n1000 packets transmitted, 340 packets received, 66% packet loss\nround-trip min\/avg\/max\/stddev = 1.354\/9.492\/404.538\/44.024 ms\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eBeing honest the pattern looks like ICMP rate-limiting.  And the Dell iDRAC sub-system is known to have as much processing power as a casio watch, so this could be a total red herring.\u003c\/p\u003e\n\n\u003cp\u003eTo that last point though, I wonder could the issue be the iDRAC modules on these boxes?  I've seen issues with those in a past life, timing out to SNMP and SSH connections, so maybe that's it.  Looking at the hosts reported above I notice they are all \u003ca href=\"\/diffusion\/ETFU\/\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eR430\u003c\/span\u003e\u003c\/a\u003e models purchased in 2015\/2016:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003cth\u003eHost\u003c\/th\u003e\u003cth\u003eModel\u003c\/th\u003e\u003cth\u003ePurchase Date\u003c\/th\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003econtint2001\u003c\/td\u003e\u003ctd\u003eDell PowerEdge \u003ca href=\"\/diffusion\/ETFU\/\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eR430\u003c\/span\u003e\u003c\/a\u003e (1U)\u003c\/td\u003e\u003ctd\u003e2016-03-24\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003egerrit2001\u003c\/td\u003e\u003ctd\u003eDell PowerEdge \u003ca href=\"\/diffusion\/ETFU\/\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eR430\u003c\/span\u003e\u003c\/a\u003e (1U)\u003c\/td\u003e\u003ctd\u003e2016-03-24\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003elogstash2021\u003c\/td\u003e\u003ctd\u003eDell PowerEdge \u003ca href=\"\/diffusion\/ETFU\/\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eR430\u003c\/span\u003e\u003c\/a\u003e\u003c\/td\u003e\u003ctd\u003e2015-12-07\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003elogstash2022\u003c\/td\u003e\u003ctd\u003eDell PowerEdge \u003ca href=\"\/diffusion\/ETFU\/\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eR430\u003c\/span\u003e\u003c\/a\u003e\u003c\/td\u003e\u003ctd\u003e2016-11-03\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ewdqs2001\u003c\/td\u003e\u003ctd\u003eDell PowerEdge \u003ca href=\"\/diffusion\/ETFU\/\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eR430\u003c\/span\u003e\u003c\/a\u003e\u003c\/td\u003e\u003ctd\u003e2016-08-15\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eI've limited time to do more checks right now.  But I think it'd be worth trying to pull a longer list of affected servers, and try to confirm the server models and then the iDRAC firmware version on each.  Possibly a firmware update, or even just an iDRAC reset on them, would help.\u003c\/p\u003e\n\n\u003cp\u003eI'll assign this to myself for now.  May not have a huge amount of time to work on it in the coming weeks but will see what I can dig up.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_192\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_193\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_194\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7394370\" id=\"7394370\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_191\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003ehashar\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7394370\" data-sigil=\"has-tooltip\" data-meta=\"0_190\"\u003e\u003cspan class=\"screen-only\"\u003eOct 1 2021, 11:27 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-10-01 11:27:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_188\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_189\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat is quite an epic diagnostic \u003ca href=\"\/p\/cmooney\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@cmooney\u003c\/span\u003e\u003c\/a\u003e ! It is definitely not trivial to end up root causing some specific piece of hardware as a common cause. Well done!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-h2zjwfqqi5cxjonrkfa7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5","contextPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},{"hovercardSpec":{"objectPHID":"PHID-REPO-lmycutf55vkt55a4bmkf"}},{"hovercardSpec":{"objectPHID":"PHID-REPO-lmycutf55vkt55a4bmkf"}},{"hovercardSpec":{"objectPHID":"PHID-REPO-lmycutf55vkt55a4bmkf"}},{"hovercardSpec":{"objectPHID":"PHID-REPO-lmycutf55vkt55a4bmkf"}},{"hovercardSpec":{"objectPHID":"PHID-REPO-lmycutf55vkt55a4bmkf"}},{"hovercardSpec":{"objectPHID":"PHID-REPO-lmycutf55vkt55a4bmkf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5","contextPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5","contextPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri","contextPHID":"PHID-TASK-bmroxwl6rw4ojm7nv5ht"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ffms22arq2fzb25pcbwn"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-h2zjwfqqi5cxjonrkfa7"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-USER-v7vgzvvcw7v2umf737ri"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"phid":"PHID-XACT-TASK-pvzphltpglloxdl"},{"phid":"PHID-XACT-TASK-u5kun325hytq7nn"},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qyz7sr5ziuo5s65ru4u5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2qjlvo4lpgqvny5yo3ic"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-guqp46cxa2rgti2tqgv5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2wquu5nwbdqcuzo5qiuf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-2wquu5nwbdqcuzo5qiuf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-l2xjewj6qlbbyrl"},{"phid":"PHID-XACT-TASK-2tgz5bfs7v26r6f"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-xtjp4x3665oyzsuiyo2l"}},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-p3wloc5uwsxwtcsqu3zw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-kd7qc5nqwsk3erf"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-asxbwersrxzrykn"},{"hovercardSpec":{"objectPHID":"PHID-USER-fo56wm4wxiwpoofn2xdu"}},{"phid":"PHID-XACT-TASK-a52xwjdbmnk4oqh"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-heihjeaiasruuvneirzh"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-xtjp4x3665oyzsuiyo2l"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-h2zjwfqqi5cxjonrkfa7"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ffms22arq2fzb25pcbwn"}},{"phid":"PHID-XACT-TASK-um7owt3f2c25ab2"},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-h2zjwfqqi5cxjonrkfa7"}},{"phid":"PHID-XACT-TASK-3hp743azl4vxsyj"},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-USER-uj5hmnml3trkvpfwvyri"}},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"phid":"PHID-XACT-TASK-xaev5tdg4n6qpmz"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ua2qbtq7oqnomik","anchor":"7110992"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-rmf4j63hvy6lrcu","anchor":"7111001"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pvzphltpglloxdl\/","ref":"T283582#7111022"},[],{"anchor":"7111022"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_70\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_71\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pvzphltpglloxdl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_72\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_73\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],[],{"tip":"Mentor","align":"E","size":300},[],{"tip":"Nerd Sniper","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-d67cimjsspywx77","anchor":"7111022"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u5kun325hytq7nn\/","ref":"T283582#7111164"},[],{"anchor":"7111164"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_85\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_86\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u5kun325hytq7nn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_87\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u5kun325hytq7nn","anchor":"7111164"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xetnguzp7f6rbdl","anchor":"7111183"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-32gsh2iq42tpqbs","anchor":"7111446"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-l2xjewj6qlbbyrl\/","ref":"T283582#7154540"},[],{"anchor":"7154540"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_100\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l2xjewj6qlbbyrl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_102\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-l2xjewj6qlbbyrl","anchor":"7154540"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2tgz5bfs7v26r6f\/","ref":"T283582#7158317"},[],{"anchor":"7158317"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_113\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2tgz5bfs7v26r6f\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_115\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2tgz5bfs7v26r6f","anchor":"7158317"},{"tip":"Via Bulk Update"},{"tip":"Silent Edit"},[],{"phid":"PHID-XACT-TASK-3nnmxdae6nyu7ug","anchor":"7167660"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xtp77do4ur2m4x3","anchor":"7204780"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kd7qc5nqwsk3erf\/","ref":"T283582#7228005"},[],{"anchor":"7228005"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_129\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kd7qc5nqwsk3erf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_131\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kd7qc5nqwsk3erf","anchor":"7228005"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-asxbwersrxzrykn\/","ref":"T283582#7348063"},[],{"anchor":"7348063"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_138\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_139\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-asxbwersrxzrykn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_140\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_141\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-asxbwersrxzrykn","anchor":"7348063"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-a52xwjdbmnk4oqh\/","ref":"T283582#7385095"},[],{"anchor":"7385095"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_151\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-a52xwjdbmnk4oqh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_153\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_154\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-a52xwjdbmnk4oqh","anchor":"7385095"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-um7owt3f2c25ab2\/","ref":"T283582#7394226"},[],{"anchor":"7394226"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_160\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_161\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-um7owt3f2c25ab2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_162\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_163\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-m74itmoimjvuv3z","anchor":"7394226"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3hp743azl4vxsyj\/","ref":"T283582#7394322"},[],{"anchor":"7394322"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_173\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_174\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3hp743azl4vxsyj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_175\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_176\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],[],{"phid":"PHID-XACT-TASK-qoex47wr4m3tqfc","anchor":"7394322"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xaev5tdg4n6qpmz\/","ref":"T283582#7394370"},[],{"anchor":"7394370"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_184\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_185\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xaev5tdg4n6qpmz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_186\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_187\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-xaev5tdg4n6qpmz","anchor":"7394370"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cffca910\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/e3ec9918\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}