for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4411205\" id=\"4411205\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_54\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003efgiunchedi\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411205\" data-sigil=\"has-tooltip\" data-meta=\"0_53\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 9:51 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 09:51:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_58\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4411214\" id=\"4411214\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_57\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_16\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/2657133\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411214\" data-sigil=\"has-tooltip\" data-meta=\"0_56\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 9:51 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 09:51:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4411216\" id=\"4411216\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_60\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003efgiunchedi\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/user-fgiunchedi\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003eUser-fgiunchedi\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411216\" data-sigil=\"has-tooltip\" data-meta=\"0_59\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 9:51 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 09:51:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_64\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4411357\" id=\"4411357\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_63\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003efgiunchedi\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/2668\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/2668\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003eDoing\u003c\/a\u003e on the \u003ca href=\"\/tag\/user-fgiunchedi\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003eUser-fgiunchedi\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411357\" data-sigil=\"has-tooltip\" data-meta=\"0_62\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 10:47 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 10:47:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4411389\" id=\"4411389\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_72\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411389\" data-sigil=\"has-tooltip\" data-meta=\"0_71\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 11:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 11:03:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_69\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_70\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOther differences with existing machines include the fact that the last batch has been installed with stretch from the get go as opposed to jessie, thus the filesystems have been formatted with \u003ctt class=\"remarkup-monospaced\"\u003exfsprogs\u003c\/tt\u003e 3.2.1 vs 4.9.0+nmu1.\u003c\/p\u003e\n\n\u003cp\u003eI took a diff of \u003ctt class=\"remarkup-monospaced\"\u003exfs_info\u003c\/tt\u003e on ms-be1039 vs ms-be1040 to get an idea of the differences below.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"diff\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gd\"\u003e-meta-data=\/dev\/sdn1              isize=512    agcount=32, agsize=30523264 blks\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+meta-data=\/dev\/sdn1              isize=512    agcount=4, agsize=244154240 blks\u003c\/span\u003e\n          =                       sectsz=512   attr=2, projid32bit=1\n\u003cspan class=\"gd\"\u003e-         =                       crc=0        finobt=0 spinodes=0 rmapbt=0\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+         =                       crc=1        finobt=1 spinodes=0 rmapbt=0\u003c\/span\u003e\n          =                       reflink=0\n\u003cspan class=\"gd\"\u003e-data     =                       bsize=4096   blocks=976744448, imaxpct=5\u003c\/span\u003e\n\u003cspan class=\"gd\"\u003e-         =                       sunit=64     swidth=64 blks\u003c\/span\u003e\n\u003cspan class=\"gd\"\u003e-naming   =version 2              bsize=4096   ascii-ci=0 ftype=0\u003c\/span\u003e\n\u003cspan class=\"gd\"\u003e-log      =internal               bsize=4096   blocks=476928, version=2\u003c\/span\u003e\n\u003cspan class=\"gd\"\u003e-         =                       sectsz=512   sunit=64 blks, lazy-count=1\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+data     =                       bsize=4096   blocks=976616960, imaxpct=5\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+         =                       sunit=0      swidth=0 blks\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+naming   =version 2              bsize=4096   ascii-ci=0 ftype=1\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+log      =internal               bsize=4096   blocks=476863, version=2\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+         =                       sectsz=512   sunit=0 blks, lazy-count=1\u003c\/span\u003e\n realtime =none                   extsz=4096   blocks=0, rtextents=0\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4411848\" id=\"4411848\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411848\" data-sigil=\"has-tooltip\" data-meta=\"0_80\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 1:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 13:06:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_78\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_79\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI have stopped swift\/rsync on \u003ctt class=\"remarkup-monospaced\"\u003ems-be1040\u003c\/tt\u003e to inspect further, taking e.g. \u003ctt class=\"remarkup-monospaced\"\u003esde\u003c\/tt\u003e and running \u003ctt class=\"remarkup-monospaced\"\u003exfs_repair -n\u003c\/tt\u003e on it mentioned a discrepancy in free blocks:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@ms-be1040:~# xfs_repair -n \/dev\/sde1\nPhase 1 - find and verify superblock...\nPhase 2 - using internal log\n        - zero log...\n        - scan filesystem freespace and inode maps...\nsb_fdblocks 4461713825, counted 166746529\n        - found root inode chunk\nPhase 3 - for each AG...\n        - scan (but don't clear) agi unlinked lists...\n        - process known inodes and perform inode discovery...\n        - agno = 0\n        - agno = 1\n        - agno = 2\n        - agno = 3\n        - process newly discovered inodes...\nPhase 4 - check for duplicate blocks...\n        - setting up duplicate extent list...\n        - check for inodes claiming duplicate blocks...\n        - agno = 0\n        - agno = 1\n        - agno = 2\n        - agno = 3\nNo modify flag set, skipping phase 5\nPhase 6 - check inode connectivity...\n        - traversing filesystem ...\n        - traversal finished ...\n        - moving disconnected inodes to lost+found ...\nPhase 7 - verify link counts...\nNo modify flag set, skipping filesystem flush and exiting.\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4411863\" id=\"4411863\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eelukey\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411863\" data-sigil=\"has-tooltip\" data-meta=\"0_83\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 1:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 13:14:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4411872\" id=\"4411872\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_93\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4411872\" data-sigil=\"has-tooltip\" data-meta=\"0_92\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 1:17 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 13:17:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_90\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_91\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_30\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eGraphs of disk usage for the remaining affected hosts:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/eswwi7txu4dfgclyzzpg\/PHID-FILE-4z5fq5ezdjrclgvwiozb\/2018-07-10-151504_851x864_scrot.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_6\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/gjorxv5sfdqc7pgz5bkt\/PHID-FILE-debqym3qamlk6bgxehgx\/preview-2018-07-10-151504_851x864_scrot.png\" width=\"216.68981481481\" height=\"220\" alt=\"2018-07-10-151504_851x864_scrot.png (864\u00d7851 px, 81 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/yol2s4eyjpggi3pgtdha\/PHID-FILE-4m22qquy7ypqcopdpjyo\/2018-07-10-151436_845x858_scrot.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_7\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/4nseafldpfb7yyugcd2n\/PHID-FILE-tsn6gergq5yqtmm3r2si\/preview-2018-07-10-151436_845x858_scrot.png\" width=\"216.66666666667\" height=\"220\" alt=\"2018-07-10-151436_845x858_scrot.png (858\u00d7845 px, 74 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/yzlgjm3ftxubua7kiwzg\/PHID-FILE-4xqbs6w2biye3usz4zze\/2018-07-10-151525_856x856_scrot.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_8\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/btzw6xcm47a5o7ftadmo\/PHID-FILE-3avjn46pregbffmpxlu4\/preview-2018-07-10-151525_856x856_scrot.png\" width=\"220\" height=\"220\" alt=\"2018-07-10-151525_856x856_scrot.png (856\u00d7856 px, 73 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4412102\" id=\"4412102\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_102\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4412102\" data-sigil=\"has-tooltip\" data-meta=\"0_101\"\u003e\u003cspan class=\"screen-only\"\u003eJul 10 2018, 2:31 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-10 14:31:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_99\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_100\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_32\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've posted to \u003ctt class=\"remarkup-monospaced\"\u003elinux-xfs\u003c\/tt\u003e mailing list to ask if someone has run into this bug before and how to debug further.\u003c\/p\u003e\n\n\u003cp\u003eRegardless of whether we can successfully debug the problem, I think it is fair to say the bug is triggered by the different default mkfs options used at format time. Thus reverting to using the  same mkfs options used in Jessie would not trigger the bug. Sadly the bug isn't always triggered and thus hard to "bisect" by reverting some options and not others.\u003c\/p\u003e\n\n\u003cp\u003eSeveral options changed from jessie to stretch, namely: metadata section \u003ctt class=\"remarkup-monospaced\"\u003eagcount\u003c\/tt\u003e from 32 to 4 (and consequently \u003ctt class=\"remarkup-monospaced\"\u003eagsize\u003c\/tt\u003e, which is its max size of 1TB). \u003ctt class=\"remarkup-monospaced\"\u003ecrc\u003c\/tt\u003e from 0 to 1 (and its related \u003ctt class=\"remarkup-monospaced\"\u003efinobt\u003c\/tt\u003e from 0 to 1). data section \u003ctt class=\"remarkup-monospaced\"\u003esunit\u003c\/tt\u003e and \u003ctt class=\"remarkup-monospaced\"\u003eswidth\u003c\/tt\u003e went from 64 to 0. naming section \u003ctt class=\"remarkup-monospaced\"\u003eftype\u003c\/tt\u003e from 0 to 1 and finally log section \u003ctt class=\"remarkup-monospaced\"\u003esunit\u003c\/tt\u003e from 64 to 0.\u003c\/p\u003e\n\n\u003cp\u003eA possible contingency plan would be as follow, for each host affected (ms-be104[0123], and in codfw too):\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eset the host's weight in swift to 0 (multiple-step and multiple-day process)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ere-format the filesystems with "known good" options\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003erestore the host's weight in swift (multiple-step and multiple-day process)\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eThis is the safest because partitions get shuffled around the cluster and redundancy invariants are kept at all times. By temporarily sacrificing redundancy from 3 copies to 2 in the cluster we can skip step 1+3 and (sequentially) reformat filesystems and let swift fill up the formatted disk(s) again, this essentially simulates the routine operation of a failed disk, for which we don't touch swift weights.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_112\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4414934\" id=\"4414934\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003eStashbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4414934\" data-sigil=\"has-tooltip\" data-meta=\"0_110\"\u003e\u003cspan class=\"screen-only\"\u003eJul 11 2018, 8:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-11 08:37:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_108\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWSIfU1pwY2u4JUTI8jB\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2018-07-11T08:37:50Z] <godog> reboot ms-be1040 to run hardware diagnostics - \u003ca href=\"\/T199198\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT199198\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4415748\" id=\"4415748\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4415748\" data-sigil=\"has-tooltip\" data-meta=\"0_119\"\u003e\u003cspan class=\"screen-only\"\u003eJul 11 2018, 1:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-11 13:20:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_117\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_36\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe thread on \u003ctt class=\"remarkup-monospaced\"\u003elinux-xfs\u003c\/tt\u003e starts here: \u003ca href=\"https:\/\/www.spinics.net\/lists\/linux-xfs\/msg20592.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/www.spinics.net\/lists\/linux-xfs\/msg20592.html\u003c\/a\u003e and the wrong free blocks count looks like it is due to a bit flip (!). I've been running diagnostic tests on ms-be1040 today (extended tests are still finishing) but so far no issues found.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_124\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dehmh3scjpmoumxznua7\/PHID-FILE-4esn6uidadgmq7ck5yod\/alphanumeric_aleo-white_M.png-_b38ba9-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MoritzMuehlenhoff\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4416543\" id=\"4416543\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_123\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eMoritzMuehlenhoff\u003c\/a\u003e assigned this task to \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\"\u003efgiunchedi\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4416543\" data-sigil=\"has-tooltip\" data-meta=\"0_122\"\u003e\u003cspan class=\"screen-only\"\u003eJul 11 2018, 5:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-11 17:21:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4418978\" id=\"4418978\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_132\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4418978\" data-sigil=\"has-tooltip\" data-meta=\"0_131\"\u003e\u003cspan class=\"screen-only\"\u003eJul 12 2018, 8:24 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-12 08:24:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_129\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYesterday the extended diagnostics on ms-be1040 took all day and revealed no problem\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ka52fxoyng3o7qnv5ucm\/PHID-FILE-vb4igyljyhsreop7iu6s\/2018-07-11-212617_1076x843_scrot.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_10\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/wd3rqvagg333gxrkpvql\/PHID-FILE-vbk43moulrnk7s3x6gn5\/preview-2018-07-11-212617_1076x843_scrot.png\" width=\"220\" height=\"172.36059479554\" alt=\"2018-07-11-212617_1076x843_scrot.png (843\u00d71 px, 61 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_142\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4419034\" id=\"4419034\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_141\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4419034\" data-sigil=\"has-tooltip\" data-meta=\"0_140\"\u003e\u003cspan class=\"screen-only\"\u003eJul 12 2018, 8:57 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-12 08:57:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_138\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_139\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYesterday I've launched an explicit run of \u003ctt class=\"remarkup-monospaced\"\u003eswift-object-auditor\u003c\/tt\u003e on all filesystems affected. So far no corrupted files have been found, note that the object-auditor runs in the background all the time anyways and has not found corruption so far either since these hosts have entered service.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_151\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4419192\" id=\"4419192\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_150\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4419192\" data-sigil=\"has-tooltip\" data-meta=\"0_149\"\u003e\u003cspan class=\"screen-only\"\u003eJul 12 2018, 9:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-12 09:46:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_147\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_148\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWSN4kx2wY2u4JUTJyfE\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2018-07-12T09:46:15Z] <godog> shut down ms-be1041 for hardware diagnostics - \u003ca href=\"\/T199198\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT199198\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_160\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4419665\" id=\"4419665\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_159\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4419665\" data-sigil=\"has-tooltip\" data-meta=\"0_158\"\u003e\u003cspan class=\"screen-only\"\u003eJul 12 2018, 1:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-12 13:05:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_156\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_157\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWSOmFiLoDEJc1hAs_bQ\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_4\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2018-07-12T13:05:06Z] <godog> run xfs_repair \/dev\/sdd1 on ms-be1043 - \u003ca href=\"\/T199198\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT199198\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_169\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dehmh3scjpmoumxznua7\/PHID-FILE-4esn6uidadgmq7ck5yod\/alphanumeric_aleo-white_M.png-_b38ba9-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MoritzMuehlenhoff\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4421966\" id=\"4421966\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eMoritzMuehlenhoff\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4421966\" data-sigil=\"has-tooltip\" data-meta=\"0_167\"\u003e\u003cspan class=\"screen-only\"\u003eJul 13 2018, 8:30 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-13 08:30:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_165\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_166\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_46\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI reviewed the kernel logs on the Swift servers we picked as canaries to test microcode updates and I noticed that we also various kernel error messages related to XFS, e.g.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eJul 12 13:38:52 ms-be1022 kernel: [11069450.080346] INFO: task xfs_db:40112 blocked for more than 120 seconds.\nJul 12 13:38:52 ms-be1022 kernel: [11069450.116920]       Not tainted 4.9.0-6-amd64 #1 Debian 4.9.82-1+deb9u3\nJul 12 13:38:52 ms-be1022 kernel: [11069450.151978] "echo 0 > \/proc\/sys\/kernel\/hung_task_timeout_secs" disables this message.\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194427] xfs_db          D    0 40112  40109 0x00000000\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194592]  ffff8b4c65d8b800 0000000000000000 ffff8b3ec521e0c0 ffff8b4e3f4d8940\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194751]  ffff8b4e38ff4080 ffff9e7866203cf8 ffffffffa0e0c649 ffff8b4e34f29400\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194923]  0000000000000246 ffff8b4e3f4d8940 ffff8b4e34f296e0 ffff8b3ec521e0c0\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194927] Call Trace:\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194938]  [<ffffffffa0e0c649>] ? __schedule+0x239\/0x6f0\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194940]  [<ffffffffa0e0cb32>] ? schedule+0x32\/0x80\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194945]  [<ffffffffa0a3197e>] ? wb_wait_for_completion+0x5e\/0x90\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194949]  [<ffffffffa08b9c50>] ? prepare_to_wait_event+0xf0\/0xf0\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194952]  [<ffffffffa0a33728>] ? __writeback_inodes_sb_nr+0x98\/0xe0\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194955]  [<ffffffffa0a39c36>] ? __sync_filesystem+0x46\/0x50\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194957]  [<ffffffffa0a39ec6>] ? sync_filesystem+0x26\/0x50\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194961]  [<ffffffffa0a41f8f>] ? fsync_bdev+0x1f\/0x50\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194966]  [<ffffffffa0b0f793>] ? blkdev_ioctl+0x5a3\/0x970\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194968]  [<ffffffffa0a419a9>] ? block_ioctl+0x39\/0x40\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194971]  [<ffffffffa0a19db2>] ? do_vfs_ioctl+0xa2\/0x620\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194975]  [<ffffffffa0a0aeff>] ? SYSC_newfstat+0x2f\/0x50\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194976]  [<ffffffffa0a1a3a4>] ? SyS_ioctl+0x74\/0x80\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194980]  [<ffffffffa0803b7f>] ? do_syscall_64+0x8f\/0xf0\nJul 12 13:38:52 ms-be1022 kernel: [11069450.194984]  [<ffffffffa0e113b8>] ? entry_SYSCALL_64_after_swapgs+0x42\/0xb0\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThese can be found on ms-be1016, 1039,  1028 and 1036 and may or may not be related, needs further investigation, might be worth to point out in the discussions on linux-xfs?\u003c\/p\u003e\n\n\u003cp\u003e(Initially I thought these could be related to the microcode updates, but I also found them on hosts which didn't get the microcode update)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_178\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4422784\" id=\"4422784\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_177\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4422784\" data-sigil=\"has-tooltip\" data-meta=\"0_176\"\u003e\u003cspan class=\"screen-only\"\u003eJul 13 2018, 2:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-13 14:28:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_174\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_175\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@MoritzMuehlenhoff\u003c\/span\u003e\u003c\/a\u003e interesting! I'll collect more info and post upstream with those too.\u003c\/p\u003e\n\n\u003cp\u003eDiagnostics on two machines haven't yielded anything wrt faulty hardware, I also tried \u003ctt class=\"remarkup-monospaced\"\u003exfs_repair\u003c\/tt\u003e on one filesystem and it fixed the issue. I'll fix most filesystems and leave two as it is for investigation for some days.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_187\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4426505\" id=\"4426505\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4426505\" data-sigil=\"has-tooltip\" data-meta=\"0_185\"\u003e\u003cspan class=\"screen-only\"\u003eJul 16 2018, 8:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-16 08:19:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_183\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWSiLEWrwY2u4JUTNr7A\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_5\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2018-07-16T08:19:27Z] <godog> run xfs_repair on filesystems reporting negative space available on ms-be1041 - \u003ca href=\"\/T199198\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT199198\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_190\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4428093\" id=\"4428093\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_189\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003efgiunchedi\u003c\/a\u003e closed subtask \u003ca href=\"\/T199436\" class=\"phui-handle handle-status-closed\"\u003eT199436: Alert on negative disk space available\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4428093\" data-sigil=\"has-tooltip\" data-meta=\"0_188\"\u003e\u003cspan class=\"screen-only\"\u003eJul 16 2018, 3:55 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-07-16 15:55:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-mdjzpmqodnzshzdgzyxe"}},[],[],[],[],{"phid":"PHID-FILE-4z5fq5ezdjrclgvwiozb","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/eswwi7txu4dfgclyzzpg\/PHID-FILE-4z5fq5ezdjrclgvwiozb\/2018-07-10-151504_851x864_scrot.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/eswwi7txu4dfgclyzzpg\/PHID-FILE-4z5fq5ezdjrclgvwiozb\/2018-07-10-151504_851x864_scrot.png","alt":"2018-07-10-151504_851x864_scrot.png (864\u00d7851 px, 81 KB)","viewable":true,"monogram":"F23554534"},{"phid":"PHID-FILE-4m22qquy7ypqcopdpjyo","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/yol2s4eyjpggi3pgtdha\/PHID-FILE-4m22qquy7ypqcopdpjyo\/2018-07-10-151436_845x858_scrot.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/yol2s4eyjpggi3pgtdha\/PHID-FILE-4m22qquy7ypqcopdpjyo\/2018-07-10-151436_845x858_scrot.png","alt":"2018-07-10-151436_845x858_scrot.png (858\u00d7845 px, 74 KB)","viewable":true,"monogram":"F23554533"},{"phid":"PHID-FILE-4xqbs6w2biye3usz4zze","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/yzlgjm3ftxubua7kiwzg\/PHID-FILE-4xqbs6w2biye3usz4zze\/2018-07-10-151525_856x856_scrot.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/yzlgjm3ftxubua7kiwzg\/PHID-FILE-4xqbs6w2biye3usz4zze\/2018-07-10-151525_856x856_scrot.png","alt":"2018-07-10-151525_856x856_scrot.png (856\u00d7856 px, 73 KB)","viewable":true,"monogram":"F23554532"},{"hovercardSpec":{"objectPHID":"PHID-TASK-mdjzpmqodnzshzdgzyxe"}},{"phid":"PHID-FILE-vb4igyljyhsreop7iu6s","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/ka52fxoyng3o7qnv5ucm\/PHID-FILE-vb4igyljyhsreop7iu6s\/2018-07-11-212617_1076x843_scrot.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/ka52fxoyng3o7qnv5ucm\/PHID-FILE-vb4igyljyhsreop7iu6s\/2018-07-11-212617_1076x843_scrot.png","alt":"2018-07-11-212617_1076x843_scrot.png (843\u00d71 px, 61 KB)","viewable":true,"monogram":"F23668273"},{"hovercardSpec":{"objectPHID":"PHID-TASK-mdjzpmqodnzshzdgzyxe"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-mdjzpmqodnzshzdgzyxe"}},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds","contextPHID":"PHID-TASK-mdjzpmqodnzshzdgzyxe"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-mdjzpmqodnzshzdgzyxe"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-eftciewhdehz7y43577l"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ei6zha7odprponwjhcjj"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-kiroiu6td6lavmyvbk4h"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-eftciewhdehz7y43577l"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-tq2wqwy66vq62mm"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-hv36eh6yqq4nhmc"},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-bt6nodcxyghw56u"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-6evkvod67umfnxv"},{"phid":"PHID-XACT-TASK-b6vzwyixh7z7qk4"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-sbek6uy6oet5jvv"},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-4w7vcdpc6dnhx3r"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-bqbsk7svgnexosp"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-2yqd2byckhv6ozb"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-cefqqfico7bpt5y"},{"phid":"PHID-XACT-TASK-ewsd32a5qbxxhyt"},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-ivl53ewqj4xxs3x"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-ztxdywjeejj2ur3"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-od3spudcstz75ud","anchor":"4411205"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-lvf5ec2m2ewztws","anchor":"4411214"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-e5bkfgjwhfee7vp","anchor":"4411216"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ey47ttpliukkeqj","anchor":"4411357"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tq2wqwy66vq62mm\/","ref":"T199198#4411389"},[],{"anchor":"4411389"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_65\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_66\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tq2wqwy66vq62mm\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_67\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_68\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tq2wqwy66vq62mm","anchor":"4411389"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hv36eh6yqq4nhmc\/","ref":"T199198#4411848"},[],{"anchor":"4411848"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_74\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_75\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hv36eh6yqq4nhmc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_76\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_77\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hv36eh6yqq4nhmc","anchor":"4411848"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-t7fcpp54r3whptu","anchor":"4411863"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bt6nodcxyghw56u\/","ref":"T199198#4411872"},[],{"anchor":"4411872"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_86\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_87\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bt6nodcxyghw56u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_88\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_89\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bt6nodcxyghw56u","anchor":"4411872"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6evkvod67umfnxv\/","ref":"T199198#4412102"},[],{"anchor":"4412102"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_95\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6evkvod67umfnxv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_97\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_98\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6evkvod67umfnxv","anchor":"4412102"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-b6vzwyixh7z7qk4\/","ref":"T199198#4414934"},[],{"anchor":"4414934"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_104\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-b6vzwyixh7z7qk4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_106\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_107\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-b6vzwyixh7z7qk4","anchor":"4414934"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-sbek6uy6oet5jvv\/","ref":"T199198#4415748"},[],{"anchor":"4415748"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_113\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-sbek6uy6oet5jvv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_115\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_116\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-sbek6uy6oet5jvv","anchor":"4415748"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-zgxxhbjbrqdoxhd","anchor":"4416543"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4w7vcdpc6dnhx3r\/","ref":"T199198#4418978"},[],{"anchor":"4418978"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_125\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_126\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4w7vcdpc6dnhx3r\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_127\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_128\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4w7vcdpc6dnhx3r","anchor":"4418978"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bqbsk7svgnexosp\/","ref":"T199198#4419034"},[],{"anchor":"4419034"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_134\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_135\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bqbsk7svgnexosp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_136\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_137\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bqbsk7svgnexosp","anchor":"4419034"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2yqd2byckhv6ozb\/","ref":"T199198#4419192"},[],{"anchor":"4419192"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_143\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_144\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2yqd2byckhv6ozb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_145\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_146\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2yqd2byckhv6ozb","anchor":"4419192"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cefqqfico7bpt5y\/","ref":"T199198#4419665"},[],{"anchor":"4419665"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_152\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_153\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cefqqfico7bpt5y\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_154\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_155\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cefqqfico7bpt5y","anchor":"4419665"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ewsd32a5qbxxhyt\/","ref":"T199198#4421966"},[],{"anchor":"4421966"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_161\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ewsd32a5qbxxhyt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_163\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_164\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ewsd32a5qbxxhyt","anchor":"4421966"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ivl53ewqj4xxs3x\/","ref":"T199198#4422784"},[],{"anchor":"4422784"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_170\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_171\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ivl53ewqj4xxs3x\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_172\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_173\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ivl53ewqj4xxs3x","anchor":"4422784"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ztxdywjeejj2ur3\/","ref":"T199198#4426505"},[],{"anchor":"4426505"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_179\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_180\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ztxdywjeejj2ur3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_181\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ztxdywjeejj2ur3","anchor":"4426505"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7xv5fakdwer35dy","anchor":"4428093"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css"]}