for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_102\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7938631\" id=\"7938631\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eMatthewVernon\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7938631\" data-sigil=\"has-tooltip\" data-meta=\"0_100\"\u003e\u003cspan class=\"screen-only\"\u003eMay 18 2022, 2:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-05-18 14:27:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7938640\" id=\"7938640\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_104\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_29\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/4771577\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7938640\" data-sigil=\"has-tooltip\" data-meta=\"0_103\"\u003e\u003cspan class=\"screen-only\"\u003eMay 18 2022, 2:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-05-18 14:27:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/giz4kiiwghycxmhpfwnm\/PHID-FILE-hoikbdz5d2t6fnwo7yxy\/bbbc81-alphanumeric_lato-dark_L.png-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/LSobanski\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7938644\" id=\"7938644\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/LSobanski\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003eLSobanski\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7938644\" data-sigil=\"has-tooltip\" data-meta=\"0_106\"\u003e\u003cspan class=\"screen-only\"\u003eMay 18 2022, 2:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-05-18 14:28:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_111\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7950020\" id=\"7950020\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_110\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eMatthewVernon\u003c\/a\u003e mentioned this in \u003ca href=\"\/T309027\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003eT309027: Poweredge R730xd, R740xd, R740xd2 SSDs not visible to OS as SSDs\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7950020\" data-sigil=\"has-tooltip\" data-meta=\"0_109\"\u003e\u003cspan class=\"screen-only\"\u003eMay 23 2022, 2:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-05-23 14:08:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_114\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8146177\" id=\"8146177\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_113\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eMatthewVernon\u003c\/a\u003e mentioned this in \u003ca href=\"\/T308644\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003eT308644: unstable device mapping of SSDs causing swift\/puppet problems - example reimage\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8146177\" data-sigil=\"has-tooltip\" data-meta=\"0_112\"\u003e\u003cspan class=\"screen-only\"\u003eAug 11 2022, 4:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-08-11 16:03:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8267946\" id=\"8267946\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003ejbond\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8267946\" data-sigil=\"has-tooltip\" data-meta=\"0_121\"\u003e\u003cspan class=\"screen-only\"\u003eSep 28 2022, 11:24 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-28 11:24:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_119\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_36\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIs there a sever we can use to test on also can you please describe exactly how you want the disks configured.  e.g.\u003c\/p\u003e\n\n\u003cp\u003esystem has two ssd's and N rotational disk.\u003c\/p\u003e\n\n\u003cp\u003eSSD's need to have three partitions\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003epartitions 1 needs:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eN%\/NGb size of the full disk\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003epart of a RAID-X array\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eused for ...\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003epartitions 2 needs:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eN%\/NGb size of the full disk\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003epart of a RAID-X array\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eused for ...\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003epartitions 3 needs:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eN%\/NGb size of the full disk\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigured in xfs (anything specific to consider here)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAll rotations disks need to be configuered with 1 partition used\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003epartition 1 needs:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e100% size\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econfigured in xfs (anything specific to consider here)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eFurther:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eis there any benefit\/preference to have this configured via puppet  instead of via e.g. partman\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecan you confirm that you have applied the \u003ctt class=\"remarkup-monospaced\"\u003econvert-ssds\u003c\/tt\u003e fix from \u003ca href=\"\/T309027\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT309027\u003c\/span\u003e\u003c\/a\u003e to all swift nodes\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_132\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8272058\" id=\"8272058\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_131\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8272058\" data-sigil=\"has-tooltip\" data-meta=\"0_130\"\u003e\u003cspan class=\"screen-only\"\u003eSep 29 2022, 10:44 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-09-29 10:44:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_128\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHi! Thanks for looking at this. To answer your questions:\u003c\/p\u003e\n\n\u003cp\u003eYep, the convert-ssds fix has been applied to all ms-* nodes (not thanos yet)\u003c\/p\u003e\n\n\u003cp\u003eI wouldn't mind moving to partman doing it; a bit Chesterton's fence, but I think the aim of having puppet do it was to make replacing drives automatic (but in practice that often needs some work to poke the RAID controller, and then maybe also to mount the new drive once puppet has done partition\/mkfs\/etc), and puppet has to know about the drives to set up swift (I think!) - we could perhaps write a script instead for the partition\/label\/mkfs\/mount. A downside of puppet doing it is that it fights with swift_drive_audit when the latter unmounts (and comments-out in fstab) a faulty drive.\u003c\/p\u003e\n\n\u003cp\u003eSSD partitioning  (and \/ and swap setup) is done by partman - \u003ctt class=\"remarkup-monospaced\"\u003emodules\/install_server\/files\/autoinstall\/partman\/custom\/ms-be.cfg\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cp\u003ems-be nodes should be set up thus:\u003c\/p\u003e\n\n\u003cp\u003eSystems have 2 ssds and N spinning rust drives (N=12 for older nodes, 24 for newer ones):\u003cbr \/\u003e\nThe ssds have four partitions:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePartition 1 is 55G\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003epart of a RAID 1 array\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eroot filesystem\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePartition 2 is 1G\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003epart of a RAID 1 array\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eswap [not sure why we RAID-1 the swap device]\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePartition 3 is 200G [as of \u003ca href=\"\/T314275\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT314275\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e]\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003exfs\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eused for the accounts & containers rings\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ePartition 4 is the rest of the SSD\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003exfs\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eused for the objects-1 ring (small,fast)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAll spinning rust configured with 1 partition used\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003epartition 1 is 100% of size\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003exfs\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eused for the object ring (the bulk of the stuff in swift)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAll the swift partitions need to be labelled suitably (or similar) so that they are correctly mounted at \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sd$drive$part\u003c\/tt\u003e e.g. \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage-sda3\u003c\/tt\u003e. Currently we use \u003ctt class=\"remarkup-monospaced\"\u003emkfs -t xfs -L ${fs_label} -m crc=1 -m finobt=0 -i size=512 ${dev}\u003c\/tt\u003e for the spinning rust (see \u003ctt class=\"remarkup-monospaced\"\u003emodules\/swift\/manifests\/init_device.pp\u003c\/tt\u003e but there was a suggestion that we revisit some of those options now we're on bullseye (cf. \u003ca href=\"\/T279637\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT279637\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_141\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8283733\" id=\"8283733\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8283733\" data-sigil=\"has-tooltip\" data-meta=\"0_139\"\u003e\u003cspan class=\"screen-only\"\u003eOct 4 2022, 4:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-04 16:25:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_137\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSorry, you also asked about servers; for probably-non-invasive tests (reboots &c), you can pretty much pick any storage host and downtime it - maybe \u003ctt class=\"remarkup-monospaced\"\u003ems-be2056\u003c\/tt\u003e for the older hw and \u003ctt class=\"remarkup-monospaced\"\u003ems-be2069\u003c\/tt\u003e for the newer? If you want to wipe a system and try from scratch, I can drain one for you, that takes ~2 weeks (but is easy to do, so please don't hesitate to ask, just bear the lead time in mind).\u003c\/p\u003e\n\n\u003cp\u003eCurrently reimaging wipes the entire SSD and leaves the spinning rust alone; that's fine (the SSD swift partitions are relatively quick to refill).\u003c\/p\u003e\n\n\u003cp\u003eDo you need anything else from me at this point?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8290272\" id=\"8290272\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8290272\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"screen-only\"\u003eOct 6 2022, 12:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-06 12:05:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_146\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_147\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eThis is a node where the installer picked a wrong drive to try and build the md-raid set for \/ onto\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSo this seems to at least be one of the main issues here.  i think if we can solve the problem of creating the root partition on the SSD's in d-i then we can probably fix the mounting\/reformating issue in puppet later.  With that in mind i wonder if we try to KISS and just offline al the rust disks when we do a rebuild.   nothing in d-i touches the rust discs so in theory we can run a cookbook that:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003edisables rust disks\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecalls the reimage script\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eenables rust disks\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eWith this method d-i will only see the two SSD disks and as such will have no way to get confused and pick the wrong disks.  does this sound like something worth exploring?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_160\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8290550\" id=\"8290550\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8290550\" data-sigil=\"has-tooltip\" data-meta=\"0_158\"\u003e\u003cspan class=\"screen-only\"\u003eOct 6 2022, 1:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-06 13:07:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_156\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_157\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ejust putting a note here.  after looking at the \u003ca href=\"https:\/\/galaxy.ansible.com\/dellemc\/openmanage\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003edell anisble module\u003c\/a\u003e it seems we should be able to change the disk (e.g. \/redfish\/v1\/Systems\/System.Embedded.1\/Storage\/RAID.Integrated.1-1\/Drives\/Disk.Bay.25:Enclosure.Internal.0-2:RAID.Integrated.1-1) \u003ctt class=\"remarkup-monospaced\"\u003e["Oem"]["Dell"]["DellPhysicalDisk"]["RaidStatus"] = 'Offline'\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cp\u003eor possibly\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e'post', "\/redfish\/v1\/Systems\/System.Embedded.1\/Oem\/Dell\/DellRaidService\/Actions\/DellRaidService.ChangePDState\n data={"TargetFQDD": drive_id, "State": Offline}\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8291911\" id=\"8291911\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8291911\" data-sigil=\"has-tooltip\" data-meta=\"0_167\"\u003e\u003cspan class=\"screen-only\"\u003eOct 6 2022, 3:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-06 15:26:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_165\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_47\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eWith this method d-i will only see the two SSD disks and as such will have no way to get confused and pick the wrong disks.  does this sound like something worth exploring?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCertainly! :) A caution, though, that we still have some HP nodes (ms-be105[1-9] ms-be205[1-6]), just to make things more difficult :-\/\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_178\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8335176\" id=\"8335176\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8335176\" data-sigil=\"has-tooltip\" data-meta=\"0_176\"\u003e\u003cspan class=\"screen-only\"\u003eOct 21 2022, 10:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-21 10:59:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_174\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 845502 had a related patch set uploaded (by MVernon; author: MVernon):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] swift: drain ms-be2050\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/845502\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/845502\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_181\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8335177\" id=\"8335177\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_180\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8335177\" data-sigil=\"has-tooltip\" data-meta=\"0_179\"\u003e\u003cspan class=\"screen-only\"\u003eOct 21 2022, 10:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-21 10:59:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_190\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8335190\" id=\"8335190\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_189\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8335190\" data-sigil=\"has-tooltip\" data-meta=\"0_188\"\u003e\u003cspan class=\"screen-only\"\u003eOct 21 2022, 11:06 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-21 11:06:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_186\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_187\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 845502 \u003cstrong\u003emerged\u003c\/strong\u003e by MVernon:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] swift: drain ms-be2050\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/845502\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/845502\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_193\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8335247\" id=\"8335247\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_192\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8335247\" data-sigil=\"has-tooltip\" data-meta=\"0_191\"\u003e\u003cspan class=\"screen-only\"\u003eOct 21 2022, 11:30 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-21 11:30:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8336153\" id=\"8336153\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_201\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8336153\" data-sigil=\"has-tooltip\" data-meta=\"0_200\"\u003e\u003cspan class=\"screen-only\"\u003eOct 21 2022, 7:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-21 19:44:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_198\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_199\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ewanted to update the task and first say sorry for the delayed progress on this task.\u003c\/p\u003e\n\n\u003cp\u003eTo progress with \u003ca href=\"\/T308677#8290550\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8290550\u003c\/span\u003e\u003c\/a\u003e i will need an out of service host where i can turn disk on and of without harming traffic, we explored using ms-be2039  but unfortunately this has already been un-racked and disks removed and as \u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MatthewVernon\u003c\/span\u003e\u003c\/a\u003e  has already started draining ms-be2050 it makes more sense to wait for that then getting dc-ops to re-rack and populate the disks.  That does mean a ~2 week delay before i can start testing, a delay completely on me, sorry.\u003c\/p\u003e\n\n\u003cp\u003ehowever even though this task is specifically about re-images, i think the more fundamental problem is getting stable, predictable disk names.  I think for re-images d-i is to stupid so something like the above is still worth exploring but i we still have issues in general with reboot and so should be look to fix that issue.\u003c\/p\u003e\n\n\u003cp\u003ewith that u mind i wondered if some udev puppet driven config files.  Used to ensure we always have stable names could help.  with this solution we do have th problem that we need the details before we can buld the server but need to build the server before we can get the details (with the current setup)\u003c\/p\u003e\n\n\u003cp\u003ehowever i think we could ease some pain if we first build the machine and have some script that collects all the disk uuid's which we then copy to hiera to ensure that the correct udev files are written.  this adds a bit of maual work when we introduce new hosts but gives us more confidence in rebooting hosts\u003c\/p\u003e\n\n\u003cp\u003eGoing forward i have been thinking more and more that part of the initial onboarding of servers should have some script that can boot the server and populate netbox, part of this could be assigning uuid's to disks, which gets propagated to hiera\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_213\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/15\/\" data-sigil=\"has-tooltip\" data-meta=\"0_211\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-eye\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8338236\" id=\"8338236\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003eVolans\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8338236\" data-sigil=\"has-tooltip\" data-meta=\"0_209\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 11:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 11:00:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_207\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_208\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_58\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOne way to achieve the above could be to store the disks in Netbox. They could be stored as inventory items of devices (\u003ca href=\"https:\/\/netbox-next.wikimedia.org\/dcim\/devices\/897\/inventory\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eexample on netbox-next\u003c\/a\u003e) that then could be exported via netbox-hiera to Puppet and have those drive the configuration of udev.\u003c\/p\u003e\n\n\u003cp\u003eIt would be ideal if we could get the info directly from Redfish, that would enable to create the data even before the first puppet run. From a quick check though I didn't find the same data, at least not on \u003ctt class=\"remarkup-monospaced\"\u003ems-be2060\u003c\/tt\u003e as the exposed data to the OS seems to be virtual and not matching those reported by Redfish:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# from lshw:\n*-disk:5\n     description: SCSI Disk\n     product: PERC H730P Mini\n     vendor: DELL\n\n# from Redfish physical disk:\n# \/redfish\/v1\/Systems\/System.Embedded.1\/Storage\/RAID.Integrated.1-1\/Drives\/Disk.Bay.5:Enclosure.Internal.0-2:RAID.Integrated.1-1\n'Identifiers': [{'DurableName': '500056b3d120b9c5', 'DurableNameFormat': 'NAA'}],\n 'Manufacturer': 'SEAGATE',\n 'MediaType': 'HDD',\n 'Model': 'ST8000NM012A 2KE',\n\n# from Redfish virtual disk:\n# \/redfish\/v1\/Systems\/System.Embedded.1\/Storage\/RAID.Integrated.1-1\/Volumes\/Disk.Virtual.5:RAID.Integrated.1-1\n'Identifiers': [],\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAs second best option we could ssh into the d-i environment during the reimage from the cookbook and gather the data to push to netbox.\u003c\/p\u003e\n\n\u003cp\u003eDo you know what happens on disk replacement? If the virtual identifiers are the same as seen from the OS side or they change?\u003cbr \/\u003e\nThis also of course will depend on how we do JBOD, between a pure JBOD and a one virtual disk per disk approach.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_223\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8338313\" id=\"8338313\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_222\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8338313\" data-sigil=\"has-tooltip\" data-meta=\"0_221\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 11:29 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 11:29:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_219\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003e'Identifiers': [{'DurableName': '500056b3d120b9c5', 'DurableNameFormat': 'NAA'}],\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eLinux mounts disks under \/dev\/disk\/by-id\/ using the wwn which i believe should be the same as this ansd we see that the SSD's are mapped to  something that looks similar\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003elrwxrwxrwx 1 root root    9 Jun 22 13:24 wwn-0x5002538e0092ba5a -> ..\/..\/sda\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba5a-part1 -> ..\/..\/sda1\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba5a-part2 -> ..\/..\/sda2\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba5a-part3 -> ..\/..\/sda3\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba5a-part4 -> ..\/..\/sda4\nlrwxrwxrwx 1 root root    9 Jun 22 13:24 wwn-0x5002538e0092ba60 -> ..\/..\/sdb\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba60-part1 -> ..\/..\/sdb1\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba60-part2 -> ..\/..\/sdb2\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba60-part3 -> ..\/..\/sdb3\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 wwn-0x5002538e0092ba60-part4 -> ..\/..\/sdb4\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ehowever this is dose not seem to be the value that we get from redfish (which is 500056b3b9076acc) so im not sure whats going on there?\u003cbr \/\u003e\n I have noticed that the disks are also mapped to a combination of the modle name and serial number which we can get from redfish\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003elrwxrwxrwx 1 root root    9 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910281 -> ..\/..\/sda\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910281-part1 -> ..\/..\/sda1\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910281-part2 -> ..\/..\/sda2\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910281-part3 -> ..\/..\/sda3\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910281-part4 -> ..\/..\/sda4\nlrwxrwxrwx 1 root root    9 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910287 -> ..\/..\/sdb\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910287-part1 -> ..\/..\/sdb1\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910287-part2 -> ..\/..\/sdb2\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910287-part3 -> ..\/..\/sdb3\nlrwxrwxrwx 1 root root   10 Jun 22 13:24 ata-MZ7LH480HBHQ0D3_S5YJNE0N910287-part4 -> ..\/..\/sdb4\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003epprint.pprint(r.request('get', '\/redfish\/v1\/Systems\/System.Embedded.1\/Storage\/RAID.Integrated.1-1\/Drives\/Disk.Bay.24:Enclosure.Internal.0-2:RAID.Integrated.1-1').json())\n 'Identifiers': [{'DurableName': '500056b3b9076acc',\n                  'DurableNameFormat': 'NAA'}],\n...\n 'Manufacturer': 'DELL',\n 'MediaType': 'SSD',\n 'Model': 'MZ7LH480HBHQ0D3',\n...\n 'SerialNumber': 'S5YJNE0N910281',\n...\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_233\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8338353\" id=\"8338353\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_232\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#8338353\" data-sigil=\"has-tooltip\" data-meta=\"0_231\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 11:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 11:40:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_229\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_230\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eif we can get the pci address for the raid card from redfish we may also be able top use \u003ctt class=\"remarkup-monospaced\"\u003e\/dev\/disk\/by-path\u003c\/tt\u003e e.g. \u003ctt class=\"remarkup-monospaced\"\u003epci-0000:18:00.0-scsi-0:0:24:0 -> ..\/..\/sda\u003c\/tt\u003e, this disk is in slot 24\u003cbr \/\u003e\n`\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_242\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8338482\" id=\"8338482\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_241\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8338482\" data-sigil=\"has-tooltip\" data-meta=\"0_240\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 12:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 12:35:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_238\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_239\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 848302 had a related patch set uploaded (by Jbond; author: jbond):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] C:swift::storage: drop unused udev rule\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/848302\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/848302\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_245\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"8338483\" id=\"8338483\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_244\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8338483\" data-sigil=\"has-tooltip\" data-meta=\"0_243\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 12:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 12:35:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_254\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8338658\" id=\"8338658\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8338658\" data-sigil=\"has-tooltip\" data-meta=\"0_252\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 1:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 13:50:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_250\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI find myself naively wondering if a script that ran on the host and provided puppet suitable facts might be simplest? It could then also e.g. know about commented-out filesystems in fstab from swift-drive-audit and so on\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_263\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8338999\" id=\"8338999\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8338999\" data-sigil=\"has-tooltip\" data-meta=\"0_261\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 3:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 15:44:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_259\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 848418 had a related patch set uploaded (by Jbond; author: jbond):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] C:swift::storage: add variable for data directory\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/848418\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/848418\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_272\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8339000\" id=\"8339000\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8339000\" data-sigil=\"has-tooltip\" data-meta=\"0_270\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 3:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 15:44:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_268\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 848419 had a related patch set uploaded (by Jbond; author: jbond):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] P:swift::storage: add new resource to format via pci path\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/848419\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/848419\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_281\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8339001\" id=\"8339001\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8339001\" data-sigil=\"has-tooltip\" data-meta=\"0_279\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 3:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 15:44:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_277\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 848420 had a related patch set uploaded (by Jbond; author: jbond):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] ms-be2050: enable disks by path configuerations\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/848420\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/848420\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_290\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8339250\" id=\"8339250\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8339250\" data-sigil=\"has-tooltip\" data-meta=\"0_288\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 4:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 16:53:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_286\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 848427 had a related patch set uploaded (by Jbond; author: jbond):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/cookbooks@master] sre.swift.audit-labels: Audit the disk labels of swift backend hosts\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/848427\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/848427\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_299\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8339287\" id=\"8339287\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8339287\" data-sigil=\"has-tooltip\" data-meta=\"0_297\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 5:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 17:02:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_295\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8338658\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8338658\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MatthewVernon\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI find myself naively wondering if a script that ran on the host and provided puppet suitable facts might be simplest?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eindeed facter may be a different way to go\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIt could then also e.g. know about commented-out filesystems in fstab from\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWhich commented out filesystems and how dose that play with puppet?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eswift-drive-audit and so on\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003ehmm what is this script i just created a simple cookbook with a similar name :).   I noticed that the current puppet config lables the disks with whaever device it has been booted with.  i.,e. \u003ctt class=\"remarkup-monospaced\"\u003e\/dev\/sda\u003c\/tt\u003e gets the label \u003ctt class=\"remarkup-monospaced\"\u003eswift-sda\u003c\/tt\u003e.  This means that after reboots its quite possible for the device and label to mismatch.  luckily puppet doesn't relabel them with the new device (which would defy the point of having labels in the first place) however it dose mean that there has been a bit of drift, the following is a list of hosts with the drives with an incorrect\/confusing lable\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\" style=\" max-height: 12em; overflow: auto;\"\u003esudo cookbook -c \/home\/jbond\/cookbook.yaml sre.swift.audit-labels            [16:49:59]\nSTART - Cookbook sre.swift.audit-labels for host ms-be[2040-2069].codfw.wmnet,ms-be[1040-1071].eqiad.wmnet\nPASS |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 100% (62\/62) [00:01<00:00, 51.96hosts\/s]\nFAIL |                                                                         |   0% (0\/62) [00:01<?, ?hosts\/s]\nThe following hosts have missmatches:\nms-be1040.eqiad.wmnet: sdm1,sdn1\nms-be1041.eqiad.wmnet: sdc1,sdd1\nms-be1042.eqiad.wmnet: sdl1,sdm1\nms-be1043.eqiad.wmnet: sdl1,sdm1\nms-be1044.eqiad.wmnet: sdf1,sdg1,sdh1,sdi1,sdj1,sdk1,sdl1,sdm1\nms-be1045.eqiad.wmnet: sdl1,sdm1\nms-be1046.eqiad.wmnet: sdf1,sdg1,sdk1,sdl1,sdm1,sdn1\nms-be1047.eqiad.wmnet: sdh1,sdi1,sdk1,sdl1\nms-be1048.eqiad.wmnet: sdc1,sdd1,sdl1,sdm1,sdn1\nms-be1049.eqiad.wmnet: sdm1,sdn1\nms-be1050.eqiad.wmnet: sdc1,sde1,sdf1,sdg1,sdh1,sdi1,sdj1,sdk1,sdl1,sdm1,sdn1\nms-be1051.eqiad.wmnet: sdi1,sdk1,sdl1,sdm1,sdn1\nms-be1052.eqiad.wmnet: sde1,sdf1,sdg1,sdl1,sdm1\nms-be1053.eqiad.wmnet: sdc1,sdd1,sde1,sdf1,sdj1,sdk1,sdm1\nms-be1054.eqiad.wmnet: sde1,sdf1,sdk1,sdl1,sdm1\nms-be1055.eqiad.wmnet: sde1,sdf1,sdi1,sdj1,sdk1,sdl1,sdm1\nms-be1056.eqiad.wmnet: sdd1,sde1\nms-be1057.eqiad.wmnet: sdc1,sdd1,sde1,sdf1,sdg1,sdh1,sdi1,sdj1,sdl1,sdm1\nms-be1058.eqiad.wmnet: sdg1,sdh1\nms-be1059.eqiad.wmnet: sdc1,sdd1,sdl1,sdn1\nms-be1061.eqiad.wmnet: sdn1,sdo1,sds1,sdt1\nms-be1062.eqiad.wmnet: sdj1,sdk1,sdn1,sdo1,sdp1,sdq1,sdt1,sdu1\nms-be1063.eqiad.wmnet: sdd1,sde1,sdh1,sdi1\nms-be1064.eqiad.wmnet: sdf1,sdg1,sdh1,sdi1,sdk1,sdl1,sdm1,sdn1,sdp1,sdq1,sdt1,sdv1\nms-be1065.eqiad.wmnet: sdh1,sdj1,sdl1,sdm1,sdv1,sdw1\nms-be1066.eqiad.wmnet: sdj1,sdk1,sdl1,sdm1,sdn1,sdt1,sdu1,sdv1\nms-be1067.eqiad.wmnet: sdc1,sdd1,sde1,sdf1,sdk1,sdm1,sdp1,sdq1,sdr1,sds1,sdu1,sdv1\nms-be1068.eqiad.wmnet: sde1,sdf1,sdg1,sdh1,sdi1,sdj1,sdl1,sdm1,sdn1,sdo1,sdp1,sdq1,sdr1,sds1,sdt1,sdv1,sdw1,sdx1,sdy1,sdz1\nms-be1069.eqiad.wmnet: sdf1,sdg1,sdh1,sdi1,sdj1,sdk1,sdl1,sdm1,sdn1,sdo1,sdp1,sdq1,sdr1,sds1,sdt1,sdv1,sdw1\nms-be1070.eqiad.wmnet: sde1,sdf1,sdk1,sdl1,sdm1,sdn1\nms-be1071.eqiad.wmnet: sdf1,sdg1\nms-be2040.codfw.wmnet: sdh1,sdi1,sdj1,sdl1,sdm1\nms-be2041.codfw.wmnet: sdi1,sdj1,sdk1\nms-be2042.codfw.wmnet: sdi1,sdj1,sdm1,sdn1\nms-be2043.codfw.wmnet: sde1,sdf1,sdj1,sdk1\nms-be2044.codfw.wmnet: sde1,sdf1,sdh1,sdi1,sdj1,sdk1,sdl1,sdm1,sdn1\nms-be2045.codfw.wmnet: sdg1,sdh1\nms-be2046.codfw.wmnet: sde1,sdf1,sdj1,sdk1,sdl1\nms-be2047.codfw.wmnet: sdg1,sdh1,sdl1,sdm1,sdn1\nms-be2048.codfw.wmnet: sde1,sdf1,sdh1,sdi1,sdl1,sdm1\nms-be2049.codfw.wmnet: sde1,sdf1,sdg1,sdh1,sdi1,sdj1,sdk1,sdl1,sdm1\nms-be2050.codfw.wmnet: sdc1,sdd1\nms-be2051.codfw.wmnet: sdc1,sdd1,sdf1,sdg1,sdj1,sdk1,sdl1,sdm1\nms-be2052.codfw.wmnet: sdg1,sdh1,sdi1,sdj1,sdk1,sdl1,sdm1\nms-be2053.codfw.wmnet: sdc1,sdd1,sde1,sdf1,sdg1,sdh1,sdi1,sdl1,sdm1\nms-be2055.codfw.wmnet: sdc1,sde1\nms-be2056.codfw.wmnet: sdc1,sdd1,sde1,sdg1\nms-be2057.codfw.wmnet: sde1,sdf1,sdl1,sdm1,sdu1,sdv1\nms-be2058.codfw.wmnet: sdk1,sdl1,sds1,sdt1,sdu1\nms-be2059.codfw.wmnet: sdc1,sdd1,sdn1,sdo1,sdq1,sdr1,sdt1,sdu1\nms-be2060.codfw.wmnet: sdd1,sde1,sdj1,sdk1,sdm1,sdn1,sdo1\nms-be2061.codfw.wmnet: sdc1,sdd1,sde1,sdf1,sdi1,sdj1\nms-be2062.codfw.wmnet: sdd1,sde1,sdj1,sdl1,sdm1,sdn1,sdo1,sdp1,sdq1,sdr1,sdt1,sdu1,sdv1,sdw1\nms-be2063.codfw.wmnet: sdf1,sdg1,sdh1,sdi1,sdk1,sdl1,sdn1,sdo1\nms-be2064.codfw.wmnet: sdd1,sde1,sdf1,sdg1,sdj1,sdl1,sdn1,sdo1,sdp1,sdq1,sdr1,sds1\nms-be2065.codfw.wmnet: sdf1,sdg1\nms-be2066.codfw.wmnet: sde1,sdf1,sdp1,sdr1\nms-be2067.codfw.wmnet: sdh1,sdj1,sdk1,sdl1,sdm1,sdn1,sdp1,sds1,sdt1,sdu1,sdv1,sdw1\nms-be2068.codfw.wmnet: sdd1,sdf1,sdp1,sdq1,sdu1,sdv1,sdw1\nms-be2069.codfw.wmnet: sdi1,sdj1,sdk1,sdl1,sdm1,sdt1,sdu1\nEND (PASS) - Cookbook sre.swift.audit-labels (exit_code=0) for host ms-be[2040-2069].codfw.wmnet,ms-be[1040-1071].eqiad.wmnet\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eGoing forward i suggest that we simply do away with all references to the sd* devices name and simply mount the drives using some stable identify e.g.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# by path\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:2:10:0 -> \/srv\/swift-storage\/pci-0000:3b:00.0-scsi-0:2:10:0\n# or by \n\/dev\/disk\/by-id\/ata-SSDSC2KB480G7R_BTYS830401AV480BGN -> \/srv\/swift-storage\/ata-SSDSC2KB480G7R_BTYS830401AV480BGN\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI have already created a few draft patches to test this (they still need work) out but im curious if there are any blockers or concerns to moving to something like this?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_308\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8339622\" id=\"8339622\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8339622\" data-sigil=\"has-tooltip\" data-meta=\"0_306\"\u003e\u003cspan class=\"screen-only\"\u003eOct 24 2022, 7:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-24 19:24:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_304\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote\u003e\u003cp\u003eluckily puppet doesn't relabel them\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI have noticed for the aux drives which use \u003ca href=\"https:\/\/github.com\/wikimedia\/puppet\/blob\/production\/modules\/swift\/manifests\/label_filesystem.pp#L11\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eswift::lable_filesystem\u003c\/a\u003e that they do relabel the disks, this means you could get some really strange behaviour if dev\/sda came up as \/dev\/sdb or the other way round\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_317\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8340698\" id=\"8340698\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8340698\" data-sigil=\"has-tooltip\" data-meta=\"0_315\"\u003e\u003cspan class=\"screen-only\"\u003eOct 25 2022, 7:44 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-25 07:44:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_313\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 848302 \u003cstrong\u003emerged\u003c\/strong\u003e by Jbond:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] C:swift::storage: drop unused udev rule\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/848302\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/848302\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_326\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8340699\" id=\"8340699\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8340699\" data-sigil=\"has-tooltip\" data-meta=\"0_324\"\u003e\u003cspan class=\"screen-only\"\u003eOct 25 2022, 7:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-25 07:48:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_322\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 849013 had a related patch set uploaded (by Jbond; author: jbond):\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] C:swift::storage: drop absented resource\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/849013\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/849013\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_335\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8340723\" id=\"8340723\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8340723\" data-sigil=\"has-tooltip\" data-meta=\"0_333\"\u003e\u003cspan class=\"screen-only\"\u003eOct 25 2022, 8:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-25 08:12:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_331\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8339622\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8339622\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jbond\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jbond\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote\u003e\u003cp\u003eluckily puppet doesn't relabel them\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI have noticed for the aux drives which use \u003ca href=\"https:\/\/github.com\/wikimedia\/puppet\/blob\/production\/modules\/swift\/manifests\/label_filesystem.pp#L11\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eswift::lable_filesystem\u003c\/a\u003e that they do relabel the disks, this means you could get some really strange behaviour if dev\/sda came up as \/dev\/sdb or the other way round\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYeah, I have sometimes found systems with two identically-labelled drives, which obviously then needs manual intervention to sort out.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_344\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8340777\" id=\"8340777\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8340777\" data-sigil=\"has-tooltip\" data-meta=\"0_342\"\u003e\u003cspan class=\"screen-only\"\u003eOct 25 2022, 8:45 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-25 08:45:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_340\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 849013 \u003cstrong\u003emerged\u003c\/strong\u003e by Jbond:\u003c\/p\u003e\n\n\u003cp class=\"remarkup-literal\"\u003e[operations\/puppet@production] C:swift::storage: drop absented resource\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/849013\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/849013\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_353\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8342204\" id=\"8342204\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8342204\" data-sigil=\"has-tooltip\" data-meta=\"0_351\"\u003e\u003cspan class=\"screen-only\"\u003eOct 25 2022, 3:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-25 15:33:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_349\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8339287\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8339287\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jbond\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jbond\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8338658\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8338658\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MatthewVernon\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eIt could then also e.g. know about commented-out filesystems in fstab from\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWhich commented out filesystems and how dose that play with puppet?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eswift-drive-audit and so on\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003ehmm what is this script i just created a simple cookbook with a similar name :).   I noticed that the current puppet config lables the disks with whaever device it has been booted with.  i.,e. \u003ctt class=\"remarkup-monospaced\"\u003e\/dev\/sda\u003c\/tt\u003e gets the label \u003ctt class=\"remarkup-monospaced\"\u003eswift-sda\u003c\/tt\u003e.  This means that after reboots its quite possible for the device and label to mismatch.  luckily puppet doesn't relabel them with the new device (which would defy the point of having labels in the first place) however it dose mean that there has been a bit of drift, the following is a list of hosts with the drives with an incorrect\/confusing lable\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003eswift-drive-audit\u003c\/tt\u003e is an \u003ca href=\"https:\/\/github.com\/openstack\/swift\/blob\/2.26.0\/bin\/swift-drive-audit\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eupstream script\u003c\/a\u003e that looks for \u003ctt class=\"remarkup-monospaced\"\u003ekern.log\u003c\/tt\u003e errors relating to swift drives and unmounts those drives (as well as commenting them out in fstab - this lacks the necessary systemd integration cf \u003ca href=\"\/T265450\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT265450\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, though upstream have \u003ca href=\"https:\/\/review.opendev.org\/c\/openstack\/swift\/+\/859861\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003emerged my fix\u003c\/a\u003e, so we might deploy that locally), although our local config has bitrotted in bullseye cf \u003ca href=\"\/T314123\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT314123\u003c\/span\u003e\u003c\/a\u003e ; I'm not sure if fixing all that will stop the proliferation of fstab entries described in \u003ca href=\"\/T265450\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT265450\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, though.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eGoing forward i suggest that we simply do away with all references to the sd* devices name and simply mount the drives using some stable identify e.g.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# by path\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:2:10:0 -> \/srv\/swift-storage\/pci-0000:3b:00.0-scsi-0:2:10:0\n# or by \n\/dev\/disk\/by-id\/ata-SSDSC2KB480G7R_BTYS830401AV480BGN -> \/srv\/swift-storage\/ata-SSDSC2KB480G7R_BTYS830401AV480BGN\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI have already created a few draft patches to test this (they still need work) out but im curious if there are any blockers or concerns to moving to something like this?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eDevices are located in the swift rings by name (e.g. \u003ctt class=\"remarkup-monospaced\"\u003esde1\u003c\/tt\u003e), which is converted to \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sde1\u003c\/tt\u003e by the \u003ctt class=\"remarkup-monospaced\"\u003edevices\u003c\/tt\u003e directive in \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/swift\/{account,container,object}-server.conf\u003c\/tt\u003e. Similarly configuration for swift-ring-manager knows about the devices on a class of servers (see \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Swift\/Ring_Management#Storage_Schemes\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eWikitech\u003c\/a\u003e for details of the latter). Given that (I assume) the ata-IDs would be different on every server, I'm not sure that's going to fit nicely with ring management...\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_362\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8345106\" id=\"8345106\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_361\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8345106\" data-sigil=\"has-tooltip\" data-meta=\"0_360\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2022, 10:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-26 10:49:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_358\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8342204\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8342204\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MatthewVernon\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003eswift-drive-audit\u003c\/tt\u003e is an \u003ca href=\"https:\/\/github.com\/openstack\/swift\/blob\/2.26.0\/bin\/swift-drive-audit\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eupstream script\u003c\/a\u003e that looks for \u003ctt class=\"remarkup-monospaced\"\u003ekern.log\u003c\/tt\u003e errors relating to swift drives and unmounts those drives (as well as commenting them out in fstab\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI would have expected puppet to interfear with that as the mount resource will also update fstab and then try top mount drives?\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eGoing forward i suggest that we simply do away with all references to the sd* devices name and simply mount the drives using some stable identify e.g.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# by path\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:2:10:0 -> \/srv\/swift-storage\/pci-0000:3b:00.0-scsi-0:2:10:0\n# or by \n\/dev\/disk\/by-id\/ata-SSDSC2KB480G7R_BTYS830401AV480BGN -> \/srv\/swift-storage\/ata-SSDSC2KB480G7R_BTYS830401AV480BGN\u003c\/pre\u003e\u003c\/div\u003e\u003c\/blockquote\u003e\u003c\/blockquote\u003e\n\n\n\n\u003cblockquote\u003e\u003cp\u003eDevices are located in the swift rings by name (e.g. \u003ctt class=\"remarkup-monospaced\"\u003esde1\u003c\/tt\u003e), which is converted to \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sde1\u003c\/tt\u003e by the \u003ctt class=\"remarkup-monospaced\"\u003edevices\u003c\/tt\u003e directive in \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/swift\/{account,container,object}-server.conf\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis bit i assume is not to much of a problem as its all controlled by puppet?  however i dont see anything referencing sde under \/etc\/swift so could be missing something\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eSimilarly configuration for swift-ring-manager knows about the devices on a class of servers (see \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Swift\/Ring_Management#Storage_Schemes\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eWikitech\u003c\/a\u003e for details of the latter). Given that (I assume) the ata-IDs would be different on every server, I'm not sure that's going to fit nicely with ring management...\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eWe could generate update things so that we generate \/etc\/swift\/hosts.yaml based on some puppetdb query.  not sure how messy this would be but it would likely create a massive hosts.yaml file which could cause other issues.  if not then i would suggest we mount then based on the slot id e.g.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# for accounts\/ssds just map the lowest number to 1 and the second to 2\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:0:12:0-part3 - \/srv\/swift-storage\/accounts1\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:0:13:0-part3 - \/srv\/swift-storage\/accounts2\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:0:12:0-part4 - \/srv\/swift-storage\/ssds1\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:0:13:0-part4 - \/srv\/swift-storage\/ssds2\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:2:5:0 -> \/srv\/swift-storage\/objects5\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:2:8:0 -> \/srv\/swift-storage\/objects8\n\/dev\/disk\/by-path\/pci-0000:3b:00.0-scsi-0:2:10:0 -> \/srv\/swift-storage\/objects10\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThis would allow a similar puppet configuration to today but moves away from labelling things based on the unstable device name of the first boot.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_371\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8345166\" id=\"8345166\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_370\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8345166\" data-sigil=\"has-tooltip\" data-meta=\"0_369\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2022, 11:07 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-26 11:07:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_367\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8345106\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8345106\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/jbond\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jbond\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8342204\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8342204\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MatthewVernon\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003eswift-drive-audit\u003c\/tt\u003e is an \u003ca href=\"https:\/\/github.com\/openstack\/swift\/blob\/2.26.0\/bin\/swift-drive-audit\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eupstream script\u003c\/a\u003e that looks for \u003ctt class=\"remarkup-monospaced\"\u003ekern.log\u003c\/tt\u003e errors relating to swift drives and unmounts those drives (as well as commenting them out in fstab\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI would have expected puppet to interfear with that as the mount resource will also update fstab and then try top mount drives?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI think \u003ctt class=\"remarkup-monospaced\"\u003eswift::mount_filesystem\u003c\/tt\u003e doesn't typically mount "new" devices (my notes on e.g. repairing drives is that a \u003ctt class=\"remarkup-monospaced\"\u003emount\u003c\/tt\u003e is often necessary after the first puppet run), but I might be wrong!\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eDevices are located in the swift rings by name (e.g. \u003ctt class=\"remarkup-monospaced\"\u003esde1\u003c\/tt\u003e), which is converted to \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sde1\u003c\/tt\u003e by the \u003ctt class=\"remarkup-monospaced\"\u003edevices\u003c\/tt\u003e directive in \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/swift\/{account,container,object}-server.conf\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis bit i assume is not to much of a problem as its all controlled by puppet?  however i dont see anything referencing sde under \/etc\/swift so could be missing something\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSorry, I wasn't very clear. As an example, \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/swift\/object-server.conf\u003c\/tt\u003e has:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003edevices = \/srv\/swift-storage\/\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo when the ring contains \u003ctt class=\"remarkup-monospaced\"\u003esde1\u003c\/tt\u003e, the swift object server knows that refers to \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sde1\u003c\/tt\u003e. But yes, you could have "objects5" instead if fstab were set up correctly (and that when replacing the drive it comes up in the same by-path location, or we have tooling to fettle fstab appropriately). [migration might be exciting]\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_380\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8345267\" id=\"8345267\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_379\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003ejbond\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8345267\" data-sigil=\"has-tooltip\" data-meta=\"0_378\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2022, 11:28 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-26 11:28:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_376\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T308677#8345166\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT308677#8345166\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MatthewVernon\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI think \u003ctt class=\"remarkup-monospaced\"\u003eswift::mount_filesystem\u003c\/tt\u003e doesn't typically mount "new" devices (my notes on e.g. repairing drives is that a \u003ctt class=\"remarkup-monospaced\"\u003emount\u003c\/tt\u003e is often necessary after the first puppet run), but I might be wrong!\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eswift::mount_filesystem uses mount with \u003ctt class=\"remarkup-monospaced\"\u003eatboot => true\u003c\/tt\u003e so it will re add a disk to fstab if its been commented out (i also manually tested this), however no need to go down that rabbit whole right now just wanted to mention it\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote\u003e\u003cblockquote\u003e\u003cp\u003eDevices are located in the swift rings by name (e.g. \u003ctt class=\"remarkup-monospaced\"\u003esde1\u003c\/tt\u003e), which is converted to \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sde1\u003c\/tt\u003e by the \u003ctt class=\"remarkup-monospaced\"\u003edevices\u003c\/tt\u003e directive in \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/swift\/{account,container,object}-server.conf\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis bit i assume is not to much of a problem as its all controlled by puppet?  however i dont see anything referencing sde under \/etc\/swift so could be missing something\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eSorry, I wasn't very clear. As an example, \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/swift\/object-server.conf\u003c\/tt\u003e has:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003edevices = \/srv\/swift-storage\/\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo when the ring contains \u003ctt class=\"remarkup-monospaced\"\u003esde1\u003c\/tt\u003e, the swift object server knows that refers to \u003ctt class=\"remarkup-monospaced\"\u003e\/srv\/swift-storage\/sde1\u003c\/tt\u003e. But yes, you could have "objects5" instead if fstab were set up correctly\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eAck\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003e(and that when replacing the drive it comes up in the same by-path location, or we have tooling to fettle fstab appropriately).\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eyes as long as someone doesn't move the pci slot the raid card is in the drives will come back up with the same path\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003e[migration might be exciting]\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eIndeed, and it may not be required.  if the confusion highlighted by sre.swift.audit-labels is acceptable then we can probably make the current labelling more stable.  If we do move to this new proposal then we could either\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFix manually in place:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003estop puppet\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eremount everything with the paths\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eupdate the ring manager to know about the new disk ->  disk naming\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003estart puppet\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAutomatically via a rebuild.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThis would obviously be a lot of work  and likely not desirable but may allow for some other changes for instance it seems a bit unnecessary to create a partition on the object disks instead of just creating one FS across the entire disk\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFix as we re-image:\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eslowly migrate systems to the newer system as we introduce new HW, rebuild or see errors\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eRight now i feel that the bigger issues are\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003esorting the reimage out\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ethe label renaming aux drives i.e.  \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T308677#8340723\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_26\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T308677#8340723\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_389\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p6ijpbhlknfqustofxh7\/PHID-FILE-q7664jjuhfkc5hy75bo6\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MatthewVernon\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"8345978\" id=\"8345978\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MatthewVernon\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003eMatthewVernon\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#8345978\" data-sigil=\"has-tooltip\" data-meta=\"0_387\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2022, 2:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2022-10-26 14:09:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_385\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe other big issue is rebooting reliably - as currently set up, puppet is unhappy if the SSDs come up as anything other than sda and sdb (and those must not be swapped); which means that to e.g. reboot all of the ms- backends for a security update someone has to watch each in turn to see if the drives came back OK and reboot again if not. And that hassle means it's not readily automatable.\u003c\/p\u003e\n\n\u003cp\u003eThe October reboots took about 2 days' FTE, and required 69 additional reboots (unevenly spread across the 62 backends).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-x2dvprbx325yhxw6mahm"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-fmp437h3novih27rouop"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ailhvar4q5wn2kx3wegl"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vgkjz7rrb6xypse6bplf"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-tihfuvjkxyypfexpgqva"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vgkjz7rrb6xypse6bplf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx","contextPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-3pikh5nbkls2pnwotgoi"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-eo4cqx5xwnf4ojlk4bbl"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-x2dvprbx325yhxw6mahm"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-6ou36bgtqunh654ag5cx"}},{"phid":"PHID-XACT-TASK-cn46trhhzd6saoh"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-ig34ncstxbgwacj"},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-cinlephlbqta5b7"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-sh73wnrxjb6oxq3"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-lqfldhqzwjb6u4y"},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-i2tdv5s4zcatf56"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-zxk4trv3c3a3eo2"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-onxq35pmzulwtwa"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-t4itsbm6q3n3vk4"},{"phid":"PHID-XACT-TASK-cqllbyc7qit2i3j"},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-tp3sv6wq7kpxfnk"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-neibtb7foylyyq3"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-kkjng3wsc22ciqv"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-byn2zjcajfvhvzt"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-3mxbey7e7s2wu3d"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-zs3xqmdm7soojug"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-5kqd3lztpgqszfe"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ln36v4q52byctin"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-pi6to6q3pb24ijv"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-dvgu2iyinui7yvy"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-az6h6vqkm3gnhqy"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-gkaeqlodxl6hje5"},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-pikgopi25wx3tgr"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ufz4hjcmmtiizan"},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-2b6cc2z6pt44utt"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-p36berwtslsbt7p"},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-5bfmvot5hmq6wrr"},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"phid":"PHID-XACT-TASK-vvxfy25xw7z6lnb"},{"hovercardSpec":{"objectPHID":"PHID-USER-64zwdzleaxmqhrsg4jkx"}},{"phid":"PHID-XACT-TASK-jsschqsajzdkxrt"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tk6l4kbcvlgjekz","anchor":"7938631"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-4an43hdfjopa27t","anchor":"7938640"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yzliibj5hqnkh2l","anchor":"7938644"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-24rzzcen66ykgpm","anchor":"7950020"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4rsv4klxwceggkk","anchor":"8146177"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-cn46trhhzd6saoh\/","ref":"T308677#8267946"},[],{"anchor":"8267946"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_115\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cn46trhhzd6saoh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cn46trhhzd6saoh","anchor":"8267946"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ig34ncstxbgwacj\/","ref":"T308677#8272058"},[],{"anchor":"8272058"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_124\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_125\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ig34ncstxbgwacj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_126\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ig34ncstxbgwacj","anchor":"8272058"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-cinlephlbqta5b7\/","ref":"T308677#8283733"},[],{"anchor":"8283733"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_133\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_134\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cinlephlbqta5b7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_135\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cinlephlbqta5b7","anchor":"8283733"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-sh73wnrxjb6oxq3\/","ref":"T308677#8290272"},[],{"anchor":"8290272"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_142\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_143\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sh73wnrxjb6oxq3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_144\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sh73wnrxjb6oxq3","anchor":"8290272"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-lqfldhqzwjb6u4y\/","ref":"T308677#8290550"},[],{"anchor":"8290550"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_151\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lqfldhqzwjb6u4y\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_153\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_154\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-lqfldhqzwjb6u4y\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lqfldhqzwjb6u4y","anchor":"8290550"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-i2tdv5s4zcatf56\/","ref":"T308677#8291911"},[],{"anchor":"8291911"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_161\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-i2tdv5s4zcatf56\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_163\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-i2tdv5s4zcatf56","anchor":"8291911"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-zxk4trv3c3a3eo2\/","ref":"T308677#8335176"},[],{"anchor":"8335176"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_170\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zxk4trv3c3a3eo2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zxk4trv3c3a3eo2","anchor":"8335176"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uoyszod7lywrdqf","anchor":"8335177"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-onxq35pmzulwtwa\/","ref":"T308677#8335190"},[],{"anchor":"8335190"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_182\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_183\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-onxq35pmzulwtwa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_184\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_185\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-onxq35pmzulwtwa","anchor":"8335190"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-6jetuz2j3wsk5jh","anchor":"8335247"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-t4itsbm6q3n3vk4\/","ref":"T308677#8336153"},[],{"anchor":"8336153"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_194\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t4itsbm6q3n3vk4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_196\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-t4itsbm6q3n3vk4","anchor":"8336153"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-cqllbyc7qit2i3j\/","ref":"T308677#8338236"},[],{"anchor":"8338236"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_203\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_204\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cqllbyc7qit2i3j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_205\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_206\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Rat Catcher","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-cqllbyc7qit2i3j","anchor":"8338236"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-tp3sv6wq7kpxfnk\/","ref":"T308677#8338313"},[],{"anchor":"8338313"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_214\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_215\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tp3sv6wq7kpxfnk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_216\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_217\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-tp3sv6wq7kpxfnk\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tp3sv6wq7kpxfnk","anchor":"8338313"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-neibtb7foylyyq3\/","ref":"T308677#8338353"},[],{"anchor":"8338353"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_224\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-neibtb7foylyyq3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-neibtb7foylyyq3\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-neibtb7foylyyq3","anchor":"8338353"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-kkjng3wsc22ciqv\/","ref":"T308677#8338482"},[],{"anchor":"8338482"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_234\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_235\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kkjng3wsc22ciqv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_236\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kkjng3wsc22ciqv","anchor":"8338482"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-myvr355a65gk2sc","anchor":"8338483"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-byn2zjcajfvhvzt\/","ref":"T308677#8338658"},[],{"anchor":"8338658"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_246\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-byn2zjcajfvhvzt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_248\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-byn2zjcajfvhvzt","anchor":"8338658"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-3mxbey7e7s2wu3d\/","ref":"T308677#8338999"},[],{"anchor":"8338999"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_255\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3mxbey7e7s2wu3d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_257\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_258\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3mxbey7e7s2wu3d","anchor":"8338999"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-zs3xqmdm7soojug\/","ref":"T308677#8339000"},[],{"anchor":"8339000"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_264\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zs3xqmdm7soojug\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_266\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zs3xqmdm7soojug","anchor":"8339000"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-5kqd3lztpgqszfe\/","ref":"T308677#8339001"},[],{"anchor":"8339001"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_273\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5kqd3lztpgqszfe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_275\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5kqd3lztpgqszfe","anchor":"8339001"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ln36v4q52byctin\/","ref":"T308677#8339250"},[],{"anchor":"8339250"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_282\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ln36v4q52byctin\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_284\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ln36v4q52byctin","anchor":"8339250"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pi6to6q3pb24ijv\/","ref":"T308677#8339287"},[],{"anchor":"8339287"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_291\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_292\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pi6to6q3pb24ijv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_293\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pi6to6q3pb24ijv","anchor":"8339287"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-dvgu2iyinui7yvy\/","ref":"T308677#8339622"},[],{"anchor":"8339622"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_300\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dvgu2iyinui7yvy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_302\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dvgu2iyinui7yvy","anchor":"8339622"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-az6h6vqkm3gnhqy\/","ref":"T308677#8340698"},[],{"anchor":"8340698"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_309\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-az6h6vqkm3gnhqy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_311\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-az6h6vqkm3gnhqy","anchor":"8340698"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-gkaeqlodxl6hje5\/","ref":"T308677#8340699"},[],{"anchor":"8340699"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_318\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gkaeqlodxl6hje5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_320\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gkaeqlodxl6hje5","anchor":"8340699"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-pikgopi25wx3tgr\/","ref":"T308677#8340723"},[],{"anchor":"8340723"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pikgopi25wx3tgr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_329\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pikgopi25wx3tgr","anchor":"8340723"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-ufz4hjcmmtiizan\/","ref":"T308677#8340777"},[],{"anchor":"8340777"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_336\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ufz4hjcmmtiizan\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_338\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ufz4hjcmmtiizan","anchor":"8340777"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-2b6cc2z6pt44utt\/","ref":"T308677#8342204"},[],{"anchor":"8342204"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_345\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2b6cc2z6pt44utt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_347\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_348\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2b6cc2z6pt44utt","anchor":"8342204"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-p36berwtslsbt7p\/","ref":"T308677#8345106"},[],{"anchor":"8345106"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_354\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_355\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-p36berwtslsbt7p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_356\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_357\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-p36berwtslsbt7p","anchor":"8345106"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-5bfmvot5hmq6wrr\/","ref":"T308677#8345166"},[],{"anchor":"8345166"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_363\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_364\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5bfmvot5hmq6wrr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_365\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_366\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5bfmvot5hmq6wrr","anchor":"8345166"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-vvxfy25xw7z6lnb\/","ref":"T308677#8345267"},[],{"anchor":"8345267"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_372\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_373\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vvxfy25xw7z6lnb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_374\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_375\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vvxfy25xw7z6lnb","anchor":"8345267"},{"targetID":"UQ0_5","uri":"\/transactions\/quote\/PHID-XACT-TASK-jsschqsajzdkxrt\/","ref":"T308677#8345978"},[],{"anchor":"8345978"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_381\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_382\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jsschqsajzdkxrt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_383\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_384\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jsschqsajzdkxrt","anchor":"8345978"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2f0e5413\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}