for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_201\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6511463\" id=\"6511463\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_200\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6511463\" data-sigil=\"has-tooltip\" data-meta=\"0_199\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 8:43 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 08:43:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_205\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6511473\" id=\"6511473\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_203\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_78\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a project: \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eSRE\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/3915188\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6511473\" data-sigil=\"has-tooltip\" data-meta=\"0_202\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 8:43 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 08:43:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_204\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7  \u003ca href=\"\/herald\/transcript\/3915188\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_214\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6511496\" id=\"6511496\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_213\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6511496\" data-sigil=\"has-tooltip\" data-meta=\"0_212\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 8:52 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 08:52:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_210\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_211\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe most logical suspect is the rollout of Varnish 6 \u003ca href=\"\/T263557\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT263557: Upgrade production cache nodes to Varnish 6\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_223\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6511513\" id=\"6511513\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_222\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6511513\" data-sigil=\"has-tooltip\" data-meta=\"0_221\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 8:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 08:58:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_219\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/000000230\/navigation-timing-by-continent?orgId=1\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eIt seems to affect North America before Europe\u003c\/a\u003e, and the timing lines up with the rollouts of codfw and esams:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/2zoibklah237mmpudvsu\/PHID-FILE-nscklonegls555v3j4ah\/Screenshot_2020-10-02_at_10.56.42.png\" class=\"phabricator-remarkup-embed-image-full\" data-sigil=\"lightboxable\" data-meta=\"0_5\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/2zoibklah237mmpudvsu\/PHID-FILE-nscklonegls555v3j4ah\/Screenshot_2020-10-02_at_10.56.42.png\" height=\"534\" width=\"1364\" alt=\"Screenshot 2020-10-02 at 10.56.42.png (534\u00d71 px, 160 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/voy6rpw7bw6bzphuejpg\/PHID-FILE-gp2s5oeb6nzwlqttfrzy\/Screenshot_2020-10-02_at_10.57.35.png\" class=\"phabricator-remarkup-embed-image-full\" data-sigil=\"lightboxable\" data-meta=\"0_6\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/voy6rpw7bw6bzphuejpg\/PHID-FILE-gp2s5oeb6nzwlqttfrzy\/Screenshot_2020-10-02_at_10.57.35.png\" height=\"550\" width=\"1360\" alt=\"Screenshot 2020-10-02 at 10.57.35.png (550\u00d71 px, 150 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eAfter the change, it stays elevated\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_226\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6511515\" id=\"6511515\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_87\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6511515\" data-sigil=\"has-tooltip\" data-meta=\"0_224\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 8:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 08:58:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_229\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6511533\" id=\"6511533\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6511533\" data-sigil=\"has-tooltip\" data-meta=\"0_227\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 9:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 09:03:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_238\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6512035\" id=\"6512035\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_237\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_89\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6512035\" data-sigil=\"has-tooltip\" data-meta=\"0_236\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 10:44 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 10:44:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_234\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_235\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_90\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've improved the per-DC\/host dashboard: \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eThe change is clearly visible on Esams on 2020-09-29 and on Eqsin on 2020-09-25, which corresponds exactly to the Varnish 6 rollout for those DCs.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_247\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6512038\" id=\"6512038\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_246\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e assigned this task to \u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6512038\" data-sigil=\"has-tooltip\" data-meta=\"0_245\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 10:45 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 10:45:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_243\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_244\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@ema\u003c\/span\u003e\u003c\/a\u003e as discussed on IRC, it seems sensible to roll back the change on at least one host on Esams for a few days next week to verify that this is what's causing the issue.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_256\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6512255\" id=\"6512255\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_255\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6512255\" data-sigil=\"has-tooltip\" data-meta=\"0_254\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 11:12 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 11:12:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_252\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6512038\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6512038\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@ema\u003c\/span\u003e\u003c\/a\u003e as discussed on IRC, it seems sensible to roll back the change on at least one host on Esams for a few days next week to verify that this is what's causing the issue.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIndeed. If, as we expect, the downgrade does bring \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eresponse-time-by-host\u003c\/a\u003e back to the previous levels, the next step is to benchmark locally \u003ca href=\"https:\/\/apt.wikimedia.org\/wikimedia\/pool\/main\/v\/varnish\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003evarnish 5.1.3-1wm15\u003c\/a\u003e vs \u003ca href=\"https:\/\/apt.wikimedia.org\/wikimedia\/pool\/component\/varnish6\/v\/varnish\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003e6.0.6-1wm1\u003c\/a\u003e and try to reproduce the regression with synthetic traffic.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_265\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oczrgb2xczu3hebwulsk\/PHID-FILE-3raolhvprjakga3u36ek\/alphanumeric_lato-white_B.png-_5880b0-255%2C255%2C255%2C0.7.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BBlack\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6512490\" id=\"6512490\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003eBBlack\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6512490\" data-sigil=\"has-tooltip\" data-meta=\"0_263\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 1:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 13:01:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_261\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eJust throwing in some random points\/counterpoints to ponder:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIt's possible it does take more than a day or three for the frontend caches to settle into an optimal pattern and compare well against the previous week, especially given there's both daily and time-of-week variations in the natural data and it's been less than a full week so far.  Do we yet have any way to distinguish the impact of cache misses when looking at this (e.g. compare ttfb increase only on synth+hit)?  Either way, a few days of rollback comparison on one host will tell us something useful about this.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWe're also battling elevated CPU utilization since the V6 upgrade in \u003ca href=\"\/T264074\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264074\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e - could be related, directly or indirectly?  It's enough to look like it bumps the host overlall utilization by a few percent in average, and we're getting up into a critical range there in general with peaks a bit above half.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDrilling into the per-host data in esams, they don't all show the pattern quite as clearly as the aggregate does.  Do their individual patterns have any correlation with per-host rollout timings at least?\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_274\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oczrgb2xczu3hebwulsk\/PHID-FILE-3raolhvprjakga3u36ek\/alphanumeric_lato-white_B.png-_5880b0-255%2C255%2C255%2C0.7.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BBlack\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6512535\" id=\"6512535\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003eBBlack\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6512535\" data-sigil=\"has-tooltip\" data-meta=\"0_272\"\u003e\u003cspan class=\"screen-only\"\u003eOct 2 2020, 1:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-02 13:12:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_270\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_98\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eEh maybe a few more to think about too:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe train this week caused a fair amount of churn with  the rollout + rollback of 1.36.0-wmf.11.  Is there any chance the train issues could also have perturbed stats enough to cause the disturbances seen here (or at least, could have delayed caches reaching optimal-ish state as fast as we'd expect)?  I don't think it ever reached group2, but I could be wrong.  A large impact on group1 might be enough to cause some issues, though.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAlso, service proxy changes on the inside.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAgain, in both cases above (and really in general), reliably splitting up any ttfb impact by generating layer (frontend synth\/hit, be synth\/hit, applayer) would be super-helpful.\u003c\/p\u003e\n\n\u003cp\u003eThe terminal-layer data in the hit\/miss graph might correlate in interesting ways and help with that, but I have no idea if anything useful is visible there yet: \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/000000500\/varnish-caching?viewPanel=1&orgId=1&refresh=15m&var-cluster=All&var-site=esams&var-status=1&var-status=2&var-status=3&var-status=4&var-status=5&from=now-7d&to=now\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana.wikimedia.org\/d\/000000500\/varnish-caching?viewPanel=1&orgId=1&refresh=15m&var-cluster=All&var-site=esams&var-status=1&var-status=2&var-status=3&var-status=4&var-status=5&from=now-7d&to=now\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_283\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516308\" id=\"6516308\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_99\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516308\" data-sigil=\"has-tooltip\" data-meta=\"0_281\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 7:56 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 07:56:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_279\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_100\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@BBlack\u003c\/span\u003e\u003c\/a\u003e we don't see elevated latency that lasts for days like that on train rollouts and rollbacks. Train rollbacks are a frequent event. We're now at 6 days of consistent higher latency across the board.\u003c\/p\u003e\n\n\u003cp\u003eWould adding the information currently exposed as x-cache-status (and in server-timing, where I can actually get it) as a third dimension on \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host?orgId=1\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethe per-host dashboard\u003c\/a\u003e be what you want?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_293\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516328\" id=\"6516328\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_292\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6516328\" data-sigil=\"has-tooltip\" data-meta=\"0_291\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 8:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 08:09:20 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_289\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_290\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_102\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAs for per-host on esams, it is very clear on every host. Even clearer if you zoom out to 30 days and switch to a 1day rolling average:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/3f37eru3n4nrbpslx7ld\/PHID-FILE-ujz6szx4jylz7vvjeur7\/Screenshot_2020-10-05_at_10.08.52.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_13\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/z335pxz3gs5yfyul7ndw\/PHID-FILE-xbtzpvtj7mskkriq632j\/preview-Screenshot_2020-10-05_at_10.08.52.png\" width=\"220\" height=\"87.588785046729\" alt=\"Screenshot 2020-10-05 at 10.08.52.png (426\u00d71 px, 77 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/fmomlq4w6pzu4mhlafoc\/PHID-FILE-xa7pwmuaoadzq2kzu4d2\/Screenshot_2020-10-05_at_10.08.10.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_14\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/7ags6iaxiuh45kvuwbgp\/PHID-FILE-7qpfbos6erigx3ehz4cs\/preview-Screenshot_2020-10-05_at_10.08.10.png\" width=\"220\" height=\"88.328664799253\" alt=\"Screenshot 2020-10-05 at 10.08.10.png (430\u00d71 px, 67 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/cvn4spewccugtvpga5xi\/PHID-FILE-rbes2l3pv7uaoozdxvgf\/Screenshot_2020-10-05_at_10.07.56.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_15\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/xdihr5sx7kisfzl3xayk\/PHID-FILE-xqmcku2rxjqxildgzxms\/preview-Screenshot_2020-10-05_at_10.07.56.png\" width=\"220\" height=\"89.349442379182\" alt=\"Screenshot 2020-10-05 at 10.07.56.png (437\u00d71 px, 78 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/4wjbiwu2muo6m6hqliub\/PHID-FILE-sov7fk4zhmin64nhip4x\/Screenshot_2020-10-05_at_10.08.26.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_16\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ykdtypboll3ztdzuzxwv\/PHID-FILE-ojawpnijiiuy3pmnwqk2\/preview-Screenshot_2020-10-05_at_10.08.26.png\" width=\"220\" height=\"87.917833800187\" alt=\"Screenshot 2020-10-05 at 10.08.26.png (428\u00d71 px, 79 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ajn4lyyx2itnuedhlptk\/PHID-FILE-bmhkh36mnihdtyxywyzm\/Screenshot_2020-10-05_at_10.08.34.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_17\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ou6e76r7jo4fvwdghimx\/PHID-FILE-le34g7ocyvxtzjh235wb\/preview-Screenshot_2020-10-05_at_10.08.34.png\" width=\"220\" height=\"88.328664799253\" alt=\"Screenshot 2020-10-05 at 10.08.34.png (430\u00d71 px, 78 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/l4khcod6ftntvtlsf4o6\/PHID-FILE-ftl6cpzyetcgrvsxtxox\/Screenshot_2020-10-05_at_10.08.42.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_18\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/2gg6l3tttsxb45zllbo4\/PHID-FILE-np5matonnobrabhttz4u\/preview-Screenshot_2020-10-05_at_10.08.42.png\" width=\"220\" height=\"87.917833800187\" alt=\"Screenshot 2020-10-05 at 10.08.42.png (428\u00d71 px, 77 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/uziwspgicriwv3gkn2zw\/PHID-FILE-2cmag5mkyj6bzzqcjyfl\/Screenshot_2020-10-05_at_10.08.03.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_19\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/vpa5s6o3rj7auaecwajh\/PHID-FILE-mybg4s74mzh4uusdadno\/preview-Screenshot_2020-10-05_at_10.08.03.png\" width=\"220\" height=\"88.739495798319\" alt=\"Screenshot 2020-10-05 at 10.08.03.png (432\u00d71 px, 78 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/slp774wqvvhcodh3nljd\/PHID-FILE-cj7moxk4buoh5pmjdszv\/Screenshot_2020-10-05_at_10.08.18.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_20\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/waa7d75ixdfchzaofnse\/PHID-FILE-5wiggsegjgdrqeoe4vcu\/preview-Screenshot_2020-10-05_at_10.08.18.png\" width=\"220\" height=\"87.50700280112\" alt=\"Screenshot 2020-10-05 at 10.08.18.png (426\u00d71 px, 77 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_296\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6516552\" id=\"6516552\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_295\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_103\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1201\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1201\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_105\"\u003eBug Reports\u003c\/a\u003e on the \u003ca href=\"\/tag\/traffic\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003eTraffic\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516552\" data-sigil=\"has-tooltip\" data-meta=\"0_294\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 9:31 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 09:31:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_305\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516644\" id=\"6516644\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_304\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_107\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516644\" data-sigil=\"has-tooltip\" data-meta=\"0_303\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 10:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 10:17:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_301\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_302\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_108\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 632201 had a related patch set uploaded (by Ema; owner: Ema):\u003cbr \/\u003e\n[operations\/puppet@production] cache: downgrade Varnish on cp3052 to 5.1.3-1wm15\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/632201\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/632201\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_308\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6516645\" id=\"6516645\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516645\" data-sigil=\"has-tooltip\" data-meta=\"0_306\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 10:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 10:17:35 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_317\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516651\" id=\"6516651\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_111\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516651\" data-sigil=\"has-tooltip\" data-meta=\"0_315\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 10:25 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 10:25:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_313\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_112\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 632201 \u003cstrong\u003emerged\u003c\/strong\u003e by Ema:\u003cbr \/\u003e\n[operations\/puppet@production] cache: downgrade Varnish on cp3052 to 5.1.3-1wm15\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/632201\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/632201\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_326\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516672\" id=\"6516672\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_113\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516672\" data-sigil=\"has-tooltip\" data-meta=\"0_324\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 10:28 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 10:28:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_322\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_114\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/eqRO-HQBpU87LSFJmbas\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2020-10-05T10:28:54Z] <ema> cp3052: depool and downgrade varnish to 5.1.3-1wm15 \u003ca href=\"\/T264398\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_335\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516680\" id=\"6516680\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_115\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516680\" data-sigil=\"has-tooltip\" data-meta=\"0_333\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 10:32 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 10:32:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_331\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_116\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/sal.toolforge.org\/log\/IqRR-HQBpU87LSFJo7mH\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2020-10-05T10:32:13Z] <ema> cp3052: pool with varnish 5.1.3-1wm15 \u003ca href=\"\/T264398\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_344\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6516730\" id=\"6516730\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_117\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516730\" data-sigil=\"has-tooltip\" data-meta=\"0_342\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 10:52 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 10:52:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_340\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_118\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eVarnish downgraded on cp3052. I've made a new dashboard comparing response time on cp3052 (v5) vs cp3054 (v6):  \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/EiAVq3FGz\/t264398?viewPanel=2&orgId=1\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana.wikimedia.org\/d\/EiAVq3FGz\/t264398?viewPanel=2&orgId=1\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/v37p5kgxunrjf6lwtmq6\/PHID-FILE-ej7ib7xjjouo7oismsg3\/Screenshot_from_2020-10-05_12-50-09.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_23\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/cvpjise4zeznor76glc2\/PHID-FILE-747hhpwrbrbesiogiqw5\/preview-Screenshot_from_2020-10-05_12-50-09.png\" width=\"220\" height=\"123.75\" alt=\"Screenshot from 2020-10-05 12-50-09.png (1\u00d72 px, 383 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eLet's wait a bit for the cache on cp3052 to refill and see.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_347\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6516782\" id=\"6516782\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_119\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_120\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6516782\" data-sigil=\"has-tooltip\" data-meta=\"0_345\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 11:10 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 11:10:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_351\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6518865\" id=\"6518865\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_349\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_121\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003eInbox, needs triage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_123\"\u003eRadar\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6518865\" data-sigil=\"has-tooltip\" data-meta=\"0_348\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 6:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 18:37:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e edited projects, added \u003ca href=\"\/project\/view\/2923\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003ePerformance-Team (Radar)\u003c\/a\u003e; removed \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_127\"\u003ePerformance-Team\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_354\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gjkyraeh6ascv7vkj3ke\/PHID-FILE-czquixv3lk45jved74s2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/CDanis\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6518936\" id=\"6518936\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_353\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/CDanis\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_128\"\u003eCDanis\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6518936\" data-sigil=\"has-tooltip\" data-meta=\"0_352\"\u003e\u003cspan class=\"screen-only\"\u003eOct 5 2020, 7:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-05 19:06:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_363\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6521346\" id=\"6521346\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_129\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6521346\" data-sigil=\"has-tooltip\" data-meta=\"0_361\"\u003e\u003cspan class=\"screen-only\"\u003eOct 6 2020, 1:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-06 13:09:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_359\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_360\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_130\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSeems like the cache is warm now and the host is faster than its peer:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/5q5qo252udtb52bq545f\/PHID-FILE-hzhq4ge43l4bqrmc3y5s\/Screenshot_2020-10-06_at_15.08.11.png\" class=\"phabricator-remarkup-embed-image-full\" data-sigil=\"lightboxable\" data-meta=\"0_24\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/5q5qo252udtb52bq545f\/PHID-FILE-hzhq4ge43l4bqrmc3y5s\/Screenshot_2020-10-06_at_15.08.11.png\" height=\"533\" width=\"1129\" alt=\"Screenshot 2020-10-06 at 15.08.11.png (533\u00d71 px, 88 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_372\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6524392\" id=\"6524392\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_131\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/mark\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003emark\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6524392\" data-sigil=\"has-tooltip\" data-meta=\"0_370\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 9:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 09:08:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_368\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_369\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt seems that this task is starting to follow the communication pattern that emerged in \u003ca href=\"\/T238494\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT238494\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, which wasn't great. Flagging it early on to try to avoid the unpleasant back and forth we had back then and did not take us very far (CC: \u003ca href=\"\/p\/mark\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@mark\u003c\/span\u003e\u003c\/a\u003e).\u003c\/p\u003e\n\n\u003cp\u003eI don't have solutions, but I see problems (from my side of course, YMMV). The pattern I see is as follows (T=Traffic, P=Performance):\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eT does some important, long overdue work\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eP notices a correlation between the work and browser-reported TTFB going up\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eP files a high-priority task with a screenshot of the graph and no analysis\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eP asks to revert some\/all of the changes if the graph does not go back to previous values\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eT changes something to see if it helps\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eP pastes a new screenshot of the graph showing that it does not\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eT gets defensive\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eBack to point 4 or 5\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eThis highlights a few problems in different areas of the \u003ca href=\"\/tag\/traffic\/\" class=\"phui-tag-view phui-tag-type-shade phui-tag-violet phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-users\" data-meta=\"0_28\" aria-hidden=\"true\"\u003e\u003c\/span\u003eTraffic\u003c\/span\u003e\u003c\/a\u003e (all of sre?) - \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-tag-view phui-tag-type-shade phui-tag-violet phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-users\" data-meta=\"0_30\" aria-hidden=\"true\"\u003e\u003c\/span\u003ePerformance-Team\u003c\/span\u003e\u003c\/a\u003e relationship, including:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eCommunication between the teams\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eExpectation management\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDefinition of responsibilities\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSome of the questions I asked\/ask myself are:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDid we explain why we had to do the work? If so, shouldn't it be clear that we're likely not going to go back to a huge pile of tech debt to make slow (TTFB ~550ms) requests 50ms faster?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhat is the job of P in the context of this issue? Is it  looking at a graph and pasting screenshots to phab? That does not seem like a valuable use of their time. Shouldn't they provide more content to the conversation? There's a ton of work that could be done, such as looking at different percentiles, different cache lookup status, digging through changelogs\/commit logs, trying synthetic benchmarks on a development environment, ... Or is that not their responsibility?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhat do phab priorities actually mean? It's clear that "unbreak now" essentially means that we must stop what we're doing and work on the UBN task (let's forget for a minute outages and other unbreak now tickets one might be working on) . But what about all other priorities? Do they reflect the relative importance for users, for the author of the task, for the assignee of the task? Should a higher priority task have precedence over lower priority ones? What if the lower priority task is OKR work?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eAre all performance regressions high priority? Does that mean that I should drop what I'm doing and investigate, as if we were in a outage? Am I supposed to do that alone?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhat is a performance regression? "Slower than a week ago for 10 minutes in a row"? One hour? Are all regressions the same? Looking at a curve and observing that it's going up or down does not seem like a sound approach.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThere's often a trade-off between reliability, performance and complexity. Which aspect do we need to prioritize?\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIs the data we're looking at even reliable? What if, like in the case of this task,  TTFB as perceived by the TLS terminator is fine, while TTFB as reported by clients is not?\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eWhen it comes to the very last point: (a) I currently see that Varnish 5 running on cp3052 does better than v6 on cp3050, cp3056, cp3058, and cp3060, while comparing v5 with v6 on cp3054, cp3062, and cp3064 I do not see anything conclusive. (b) Even looking at a node where browser-reported TTFB is much better on v5 (516 vs 605 ms):\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/hmfabls54bbpsket25pp\/PHID-FILE-gnj6nje3kcaleavb7oqo\/Screenshot_from_2020-10-07_10-41-40.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_26\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/25d7lnrhlyjsxm3cahbl\/PHID-FILE-jahgcya3z2h6qvghepvj\/preview-Screenshot_from_2020-10-07_10-41-40.png\" width=\"220\" height=\"123.75\" alt=\"Screenshot from 2020-10-07 10-41-40.png (1\u00d72 px, 229 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eBy zooming into TTFB as reported by ats-tls I see \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/EiAVq3FGz\/t264398?orgId=1&from=1602047816444&to=1602060063748&var-host=cp3050\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethe following\u003c\/a\u003e:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ehit-front: 33ms for both\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ehit-local: 34ms for both\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eint-front: ~34ms for both\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003emiss\/pass: v5 333ms, v6 is faster (!) 326ms\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSo, do we actually have a performance issue on Varnish 6?\u003c\/p\u003e\n\n\u003cp\u003eWhat I would love us to have is better communication between the teams, and (high priority ;)) a well-defined SLO based on reliable metrics, clear rules for who should do what, and when.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_381\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6524473\" id=\"6524473\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_380\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6524473\" data-sigil=\"has-tooltip\" data-meta=\"0_379\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 9:38 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 09:38:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_377\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_378\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt seems normal to me to paste screenshots of findings, create tasks when a significant regression is witnessed, verify the cause via a targeted rollback. Performance regressions are always complicated to analyse due to the organic nature of our traffic and the fact that our systems and code are constantly evolving. It's not the Performance team's responsibility to make decisions about what the Traffic team will do about identified performance regressions, merely to seek to fully understand its nature, magnitude and potential causes. When those regressions don't get addressed as a deliberate decision, we also have to keep track of them long term.\u003c\/p\u003e\n\n\u003cp\u003eI really don't understand what I did wrong here, I helped improve the per host dashboard so we could see things better, I offered Brandon to build more instrumentation that could help (and got no response). I spent hours digging through the data to get the best understanding possible. In fact I spent a lot of time verifying my claim before even creating the task, I work very hard to not bother people about false positives.\u003c\/p\u003e\n\n\u003cp\u003eWhen someone introduces a performance regression in MediaWiki code, I try to locate the source of the problem exactly the same way, involve people who likely caused it so they can verify that their code was the source of it and they make a decision about what to do. Most recent example: \u003ca href=\"\/T263832\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT263832: Major performance regression on mobile site associated with 1.36.0-wmf.10 \u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e  This involved code that people actually wrote themselves, and they don't complain about being pinged about it, we quickly worked on a fix together.\u003c\/p\u003e\n\n\u003cp\u003eI understand that it's not a comfortable position to deploy software you don't write and discover that it potentially introduces problems like this, but I never at any point in this task requested that this upgrade be cancelled or reverted globally, the point of the host revert is to narrow down the investigation, which is complicated on its own. I never blamed you or your team for anything negative, I've only left factual comments trying to figure out what's going on.\u003c\/p\u003e\n\n\u003cp\u003eCan't we just focus on the data and keep trying to track this down and coming up with potential solutions if there are any? That's all I'm trying to do here.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_391\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6524537\" id=\"6524537\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_390\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6524537\" data-sigil=\"has-tooltip\" data-meta=\"0_389\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 10:15 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 10:15:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_387\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_388\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOn the topic of reliable metrics, our RUM monitoring is reliable, battle-tested and the ultimate truth about the performance users really experience. While it can't tell us where the issue comes from between the user's device and the bits coming from our backends, when the effect is so big, it's not noise. Something is going on, if you're not finding it in ats-tls-measured latency, it means that there are more rocks to turn to find the source, not that it's a fluke.\u003c\/p\u003e\n\n\u003cp\u003eGoogle collects the same RUM metrics to make an assessment about our performance (that's their system called CRuX), which they work into their search rankings. They've declared publicly that web performance will gradually get more weight in their ranking considerations in the future. A 10% regression across the board is a big deal. It's the kind of extra latency that can make us go into a lower bucket of quality of experience from Google's perspective. Our Google search ranking is Wikipedia's lifeline, the vast majority of our traffic comes from Google. If we start sinking in their rankings, it will have devastating effects on Wikipedia. So when our RUM performance gets significantly worse, it's potential fuel for a serious problem down the line and it has to be taken seriously. It can be challenging to figure out what on our end causes those RUM changes, and that's what we're doing here. As far as I'm concerned, we've reached a major point in the investigation because we've reproduced the problem (or its inverse, I guess) on a specific host. This means that we can stop looking for wild external explanations, it's extremely likely to be something coming from our own infrastructure.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003ewhile comparing v5 with v6 on cp3054, cp3062, and cp3064 I do not see anything conclusive\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis is what I see computing the difference between cp3052 and those hosts with a 1d sampling size\/rolling average over the last 10 days:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/7pkyghkhf2tlvqf4eeeb\/PHID-FILE-rxqsvkfoxriytd2febjz\/Screenshot_2020-10-07_at_12.09.54.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_33\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/clxpph7zdxsbjfie4voj\/PHID-FILE-dsko73qavhn3tkj2eyjy\/preview-Screenshot_2020-10-07_at_12.09.54.png\" width=\"220\" height=\"189.71580817052\" alt=\"Screenshot 2020-10-07 at 12.09.54.png (971\u00d71 px, 143 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/zqrpzgzxdlmsp2u6hbsx\/PHID-FILE-moxp2txy3uyup3zvhizv\/Screenshot_2020-10-07_at_12.10.47.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_34\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/qceaxj5uupklpwhyfc7x\/PHID-FILE-5hxobg25zedw67mlegl7\/preview-Screenshot_2020-10-07_at_12.10.47.png\" width=\"220\" height=\"112.83052351375\" alt=\"Screenshot 2020-10-07 at 12.10.47.png (578\u00d71 px, 69 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ujyohhaeke4nlgzhu43w\/PHID-FILE-4x4qmlvg5micc75h7hkv\/Screenshot_2020-10-07_at_12.11.53.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_35\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/noi762yz67tgmyzicpq4\/PHID-FILE-linzh5wcmjwhfh6slymx\/preview-Screenshot_2020-10-07_at_12.11.53.png\" width=\"220\" height=\"113.11504424779\" alt=\"Screenshot 2020-10-07 at 12.11.53.png (581\u00d71 px, 70 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eThis is the same kind of magnitude of negative difference as with all the other hosts you mentioned.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_394\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/sdb5e3nkynpf4swmwiza\/PHID-FILE-cvbkyayzlwbwsqgg7q4f\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/greg\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6525611\" id=\"6525611\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_393\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/greg\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003egreg\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6525611\" data-sigil=\"has-tooltip\" data-meta=\"0_392\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 3:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 15:12:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_398\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6525955\" id=\"6525955\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_396\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_139\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e renamed this task from \u003cspan class=\"phui-timeline-value\"\u003eElevated latency starting 2020-09-28\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003e8-10% response start regression (Varnish 5.1.3-1wm15 -> 6.0.6-1wm1)\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6525955\" data-sigil=\"has-tooltip\" data-meta=\"0_395\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 4:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 16:00:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_397\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-rszp2lnrclfirxu\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_407\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6525968\" id=\"6525968\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_406\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_141\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6525968\" data-sigil=\"has-tooltip\" data-meta=\"0_405\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 4:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 16:03:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_403\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_142\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@ema\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/p\/BBlack\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@BBlack\u003c\/span\u003e\u003c\/a\u003e before I build it, I want to confirm that some complimentary information you're looking for is the ability to break down the RUM response start metric by "hit-front", "hit-local", etc. response type in addition to DC and host.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_410\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6526044\" id=\"6526044\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_409\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_143\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e updated the task description. \u003ca href=\"\/transactions\/detail\/PHID-XACT-TASK-kqjqvfwbukqebut\/\" data-sigil=\"workflow\"\u003e(Show Details)\u003c\/a\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6526044\" data-sigil=\"has-tooltip\" data-meta=\"0_408\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 4:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 16:19:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_419\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oczrgb2xczu3hebwulsk\/PHID-FILE-3raolhvprjakga3u36ek\/alphanumeric_lato-white_B.png-_5880b0-255%2C255%2C255%2C0.7.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BBlack\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6526187\" id=\"6526187\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_418\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_144\"\u003eBBlack\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6526187\" data-sigil=\"has-tooltip\" data-meta=\"0_417\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 4:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 16:49:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_415\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_416\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_145\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6525968\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6525968\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@ema\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/p\/BBlack\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@BBlack\u003c\/span\u003e\u003c\/a\u003e before I build it, I want to confirm that some complimentary information you're looking for is the ability to break down the RUM response start metric by "hit-front", "hit-local", etc. response type in addition to DC and host.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYes, I think those would be very enlightening dimensions on the data, for sure.  I think it's probably also useful to look at more than just p75.  Is the effect more or less pronounced at the p90 or p99 level? What about the p50\/p10 end of things, or even the simple mean?\u003c\/p\u003e\n\n\u003chr class=\"remarkup-hr\" \/\u003e\n\n\u003cp\u003eBut more importantly, I'd like to rewind to the meta-conversation \u003ca href=\"\/p\/ema\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@ema\u003c\/span\u003e\u003c\/a\u003e was trying to start, and maybe take a different angle of attack:\u003c\/p\u003e\n\n\u003cp\u003eThe crux of the issue is that, as things stand today, we don't have any clearly-defined \u003ca href=\"https:\/\/en.wikipedia.org\/wiki\/Service-level_objective\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eSLO\u003c\/a\u003e for the \u003ca href=\"\/tag\/traffic\/\" class=\"phui-tag-view phui-tag-type-shade phui-tag-violet phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-users\" data-meta=\"0_43\" aria-hidden=\"true\"\u003e\u003c\/span\u003eTraffic\u003c\/span\u003e\u003c\/a\u003e layer that all stakeholders are aligned on.  There are a lot of metrics, and some of them can provide hints that something at some layer may be in need of some adjustment, to some subjective measure or standard.  That any given metric data is objective and factual in nature, at its own level, is only the beginning of the conversation about that data, not the end of it.\u003c\/p\u003e\n\n\u003cp\u003eWhich of these metrics say something important from the user perspective, and which do not?  Which help us identify the pieces and layers of our infrastructure responsible for a metric moving in an undesirable direction?  What is a reasonable standard for how those metrics should behave?  From the \u003ca href=\"\/tag\/traffic\/\" class=\"phui-tag-view phui-tag-type-shade phui-tag-violet phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-users\" data-meta=\"0_45\" aria-hidden=\"true\"\u003e\u003c\/span\u003eTraffic\u003c\/span\u003e\u003c\/a\u003e perspective, I think we stlil don't even have reliable metrics that are specific to our layers of the stack (e.g. a metric for the latency impact of just our traffic edge layer of the infrastructure, in isolation from everything else, which can say something like "the p50 +latency in the traffic layers is consistently ~0.5ms, but the p95 jumped from 63ms to 340ms starting yesterday").  Once we had one or more such highly-specific metrics, then the next task would be deciding on a SLO we can all agree on, e.g. : "p95 request processing latency in the traffic layers should be in the range of 3-12 milliseconds".\u003c\/p\u003e\n\n\u003cp\u003eWhat we have today (if you'll allow a little hypberbolic story-telling license on my part) is a highly subjective reading of the tea leaves.  The tea leaves in question may be objective metric data in some sense, but there is a ton of different data we can look at, and we don't easily know which components of our stack are implicated by a change in any of the data, we're not tracking every change (good or bad), we don't have an objective standard about the acceptability of any change, etc.  The stories we end up with are basically of the nature "I noticed some graph somewhere, which can be affected by many things, seems to have diverged from normalcy to some degree, and I think I can guess at a correlated event in your layer which might have been causative, and I think this whole incident rises to some subjective level that it deserves attention and\/or fixing".\u003c\/p\u003e\n\n\u003cp\u003eI don't doubt that smart people can in fact find important things to fix by reading tea leaves like this some of the time (we've done a lot of it in SRE over the years!), but it can be frustratingly subjective in every way (interpretation, prioritization, etc), especially when it crosses inter-team barriers and there's little shared context between the teams.  I think a really good use of our time in the coming quarter(s) would be to start figuring out how to define the reliable metrics we need, and then merging this into the OKR work that's coming up at the broader SRE level about defining real SLOs we can agree on and rely on in general.  If we solve that problem, all the rest of this melts away and becomes pretty trivial and non-combative because the subjectivity has been stripped from the process.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_428\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6527005\" id=\"6527005\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_427\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_146\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6527005\" data-sigil=\"has-tooltip\" data-meta=\"0_426\"\u003e\u003cspan class=\"screen-only\"\u003eOct 7 2020, 8:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-07 20:13:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_424\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_425\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_147\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI'm happy to work with the Traffic team to increase knowledge on web performance, which metrics matter and why, help you define your own SLOs, etc. We'll also have a professional trainer in February give another round of long form in-depth training that people from your team will be able to sign up for.\u003c\/p\u003e\n\n\u003cp\u003eIn the meantime I would appreciate it if we could work together as peers and stop belittling each others\u2019 expertise by calling it "reading tea leaves" and "highly subjective". The Performance team and I have an excellent track record of finding the root cause of performance regressions, confirmed by fixes that followed.\u003c\/p\u003e\n\n\u003cp\u003eI would like this task to remain focused on what it should be: fully understanding a confirmed significant performance regression caused by a version upgrade. Once it's fully understood, you can make your own assessment about what to do with it.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_437\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6527983\" id=\"6527983\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_436\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_148\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6527983\" data-sigil=\"has-tooltip\" data-meta=\"0_435\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2020, 6:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-08 06:46:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_433\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_434\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_149\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've added a dropdown to pick the percentile on \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eHere's what it looks like for cp3052 over the period of interest:\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/6nhezxtqndshu5qu3gq3\/PHID-FILE-vvsmwngpykn3gxkhtju6\/Screenshot_2020-10-08_at_08.44.41.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_47\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/2uv72h234se32vewj3jw\/PHID-FILE-e7vqkxvqcigmqh3hctvd\/preview-Screenshot_2020-10-08_at_08.44.41.png\" width=\"220\" height=\"71.000935453695\" alt=\"Screenshot 2020-10-08 at 08.44.41.png (345\u00d71 px, 60 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/xu2ahlmkyrq22hxaa5nn\/PHID-FILE-pzttkzxf5z7kji4d5a6v\/Screenshot_2020-10-08_at_08.45.00.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_48\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ldsfovxazgvqhdsp223r\/PHID-FILE-f2oa622ws42iy42e6gdk\/preview-Screenshot_2020-10-08_at_08.45.00.png\" width=\"220\" height=\"70.581613508443\" alt=\"Screenshot 2020-10-08 at 08.45.00.png (342\u00d71 px, 62 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/mapnzqw2epnn7tykxlnk\/PHID-FILE-ltoeng6hlgfiliavfmzn\/Screenshot_2020-10-08_at_08.45.10.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_49\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/if2a2v3qutbk3qwbqnga\/PHID-FILE-7l64opfvmpc5m7vfr54j\/preview-Screenshot_2020-10-08_at_08.45.10.png\" width=\"220\" height=\"71.267605633803\" alt=\"Screenshot 2020-10-08 at 08.45.10.png (345\u00d71 px, 53 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/bklwp4yhia6b7emgeslo\/PHID-FILE-5udkcftvxkt2brxzxhjw\/Screenshot_2020-10-08_at_08.44.53.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_50\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/4h5ikz7dsbj2s4bazxyk\/PHID-FILE-eus5phgn4273wfo4m5ix\/preview-Screenshot_2020-10-08_at_08.44.53.png\" width=\"220\" height=\"71.127819548872\" alt=\"Screenshot 2020-10-08 at 08.44.53.png (344\u00d71 px, 60 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eI've also made the comparison window configurable, as a 7 days comparison starts being misleading now that the regression is more than 7 days old.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_446\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6528126\" id=\"6528126\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_445\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_150\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6528126\" data-sigil=\"has-tooltip\" data-meta=\"0_444\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2020, 8:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-08 08:23:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_442\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_443\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_151\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6524473\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6524473\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI really don't understand what I did wrong here\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIt was never my intention to offend you, and given that I did, I want to apologize. I know that you're in good faith. I know that you're a professional with extensive experience. I know that you know what you're doing.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eCan't we just focus on the data and keep trying to track this down and coming up with potential solutions if there are any? That's all I'm trying to do here.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eFocusing on the technical aspects is the opposite of what I want to do. Sure, we can work on the technical issue. Sure, this task is not the right place to discuss team dynamics, expectation management, and responsibilities. However, I think that the problems mentioned in \u003ca href=\"\/T264398#6524392\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6524392\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e are real, and important. I'm happy to move the discussion anywhere else, but I don't know where.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_456\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6528203\" id=\"6528203\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_455\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_152\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6528203\" data-sigil=\"has-tooltip\" data-meta=\"0_454\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2020, 9:03 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-08 09:03:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_452\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_153\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI don't think that this discussion is appropriate in a public forum. An email thread seems like an ok starting point, and\/or a meeting. Your team needs a mailing list, emails I send to Brandon remain without a reply half the time. Last example dated September 16. Maybe because my reports are dismissed wholesale by default and as such don't deserve the courtesy of a response, I suppose.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_459\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/lvt5pyizsk3uckl4msuc\/PHID-FILE-eiyvx2uovr7zblyb6rkl\/profile-Kosiaris%2C_Alexandros_Sept_2013.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/akosiaris\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6528373\" id=\"6528373\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/akosiaris\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003eakosiaris\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6528373\" data-sigil=\"has-tooltip\" data-meta=\"0_457\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2020, 10:01 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-08 10:01:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_468\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2nco5z2tyvhf6dfxheoj\/PHID-FILE-pacxilbspughn725akdz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mark\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6528886\" id=\"6528886\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_467\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mark\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_155\"\u003emark\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6528886\" data-sigil=\"has-tooltip\" data-meta=\"0_466\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2020, 11:52 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-08 11:52:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_464\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_465\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHi all,\u003c\/p\u003e\n\n\u003cp\u003eI recommend we limit the conversations on this task to the technical aspects of this particular regression and its investigation, going forward. I will followup with everyone on the discussion on team dynamics, expectations and procedures in private, by email and in meetings. Thanks! :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_477\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6531798\" id=\"6531798\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_476\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_157\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6531798\" data-sigil=\"has-tooltip\" data-meta=\"0_475\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2020, 8:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-09 08:33:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_473\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_474\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_158\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI would like to know what the Traffic team is doing or planning on doing about this investigation at the moment. Now that I've narrowed it down to this Varnish update with very high confidence, this reaches the limits of what I can find on my own. The Traffic team been driving this update and has in-depth knowledge of Varnish and the edge cache stack, their expertise is required to narrow it down further.\u003c\/p\u003e\n\n\u003cp\u003eMy actions and findings are clearly documented here, I'm working to gather more information with \u003ca href=\"\/T264987\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264987: Add cache response type and response size as new dimensions to navtiming_responsestart_by_host_seconds prometheus metric\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e which is going to take some time to get reviewed, merged and deployed. Is this extra information blocking the Traffic team from further investigating what changed in Varnish 6 that's causing this issue? Is anyone still trying to fix \u003ca href=\"\/T264074\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264074: varnishkafka 1.1.0 CPU usage increase\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e with the hope that it will make this regression go away at the same time?\u003c\/p\u003e\n\n\u003cp\u003eThe Traffic team's investigation shouldn't stop at the first other source of data that suggests that nothing has changed, like self-reported ats-tls latency. This regression affects all our frontend metrics: \u003ca href=\"https:\/\/performance.wikimedia.org\/#!\/year\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/performance.wikimedia.org\/#!\/year\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eVarnish 6 as it's been deployed makes the experience worse for our users. Now that the regression has been going on for a while, we can also see that it has caused a dip \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/000000551\/performance-perception-survey\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ein user satisfaction ratio\u003c\/a\u003e (this is measured with a survey asking thousands of users their opinion daily):\u003c\/p\u003e\n\n\u003cp\u003e\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/yuykvqyhwkb6wmwnuzvm\/PHID-FILE-ovn76whgqfzphooz7lfq\/Screenshot_2020-10-09_at_09.46.33.png\" class=\"phabricator-remarkup-embed-image-full\" data-sigil=\"lightboxable\" data-meta=\"0_57\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/yuykvqyhwkb6wmwnuzvm\/PHID-FILE-ovn76whgqfzphooz7lfq\/Screenshot_2020-10-09_at_09.46.33.png\" height=\"267\" width=\"533\" alt=\"Screenshot 2020-10-09 at 09.46.33.png (267\u00d7533 px, 45 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eThat's a 2% reduction in people who think that pages load "fast enough". Meaning that the effect is definitely big enough for users to notice.\u003c\/p\u003e\n\n\u003cp\u003eDebating whether this is a magnitude of regression we can afford to make permanent when balanced with other factors should happen after the regression is fully understood and reasonable attempts have been made at fixing it if we have any leads. The Performance team can inform the debate of whether we can afford such a regression being made permanent and the Traffic team, as owners of the systems involved, will decide what to do and will be accountable for the final decision.\u003c\/p\u003e\n\n\u003cp\u003eCurrently, however, I don't have confirmation that the Traffic team has a good understanding as to why this is happening and I see no updates indicating where and how they're looking to figure this out. Maybe the fix is as simple as a new configuration setting in Varnish 6. But we're not getting any information here that suggests that the Traffic team is looking at things of that nature. Maybe the Traffic team doesn't know how to figure out that Varnish 6 is causing this and is out of ideas, in which case it should be stated, so that everyone else can know whether we've reached the limits of what can be found out here. It helps to share which dead ends have been encountered as well, in order to reduce duplicated efforts and maybe come up with alternative ideas on what to look into.\u003c\/p\u003e\n\n\u003cp\u003eIf the Traffic team is denying that anything is wrong, which also seems like a possibility given the nature of the few responses we've seen on this task that were actually about the subject matter, then Traffic has to explain what we're seeing on cp3052, why all frontend metrics are going up across the board as a result of this update (as demonstrated by extremely strong temporal correlation to the staged deployment) and why user sentiment is dropping.\u003c\/p\u003e\n\n\u003cp\u003eIt's not for every web performance regression that we get such a strong signal pointing in a single direction. It is however pretty common to see conflicting data when trying to look deeper inside our infrastructure, like when you're looking at ats-tls-reported latency. That's generally when we find that this internal monitoring is only a partial view, is possibly incorrect or misleading and that maybe we're not measuring some things that we should be. There should be some actionables filed as subtasks if the Traffic team identifies blind spots in the internal monitoring of the edge cache. Besides \u003ca href=\"\/T264987\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264987\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, are there things we're not currently measuring that the Traffic team thinks could help with this investigation?\u003c\/p\u003e\n\n\u003cp\u003eLastly, Varnish 6.0 is 5 minor versions behind the current latest stable. Has upgrading Varnish further been considered? Maybe there are too many breaking VCL changes to upgrade to 6.1 easily as an experiment, but it would be helpful to know if it's been looked into. A further upgrade is something we could try out on cp3052, for example. If the current investigation has reached a dead end for the Traffic team, this is probably a good candidate for a next step, hoping that we're currently hitting a bug that's been fixed in later versions.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_486\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6532701\" id=\"6532701\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_159\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6532701\" data-sigil=\"has-tooltip\" data-meta=\"0_484\"\u003e\u003cspan class=\"screen-only\"\u003eOct 9 2020, 3:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-09 15:13:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_482\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_483\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_160\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWith \u003ca href=\"https:\/\/github.com\/rakyll\/hey\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehey\u003c\/a\u003e on cp3052 (Varnish 5.1.3-1wm15) and cp3054 (6.0.6-1wm1) I obtained the following two latency distributions in milliseconds when benchmarking cache hits for enwiki's Main_page (~80k in size):\u003c\/p\u003e\n\n\u003cp\u003e\u003ctt class=\"remarkup-monospaced\"\u003e.\/hey -n 1000 -c 10 -H "X-Forwarded-Proto: https" -host "en.wikipedia.org" http:\/\/localhost\/wiki\/Main_Page\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e5.1.3-1wm15\u003c\/td\u003e\u003ctd\u003e6.0.6-1wm1\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep10\u003c\/td\u003e\u003ctd\u003e1.6\u003c\/td\u003e\u003ctd\u003e1.6\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003e2.0\u003c\/td\u003e\u003ctd\u003e2.0\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep50\u003c\/td\u003e\u003ctd\u003e2.5\u003c\/td\u003e\u003ctd\u003e2.4\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003e3.2\u003c\/td\u003e\u003ctd\u003e3.1\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep90\u003c\/td\u003e\u003ctd\u003e4.9\u003c\/td\u003e\u003ctd\u003e4.7\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep95\u003c\/td\u003e\u003ctd\u003e6.6\u003c\/td\u003e\u003ctd\u003e6.7\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep99\u003c\/td\u003e\u003ctd\u003e10.9\u003c\/td\u003e\u003ctd\u003e11.5\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eLatencies for the page about \u003ca href=\"https:\/\/en.wikipedia.org\/wiki\/U.C._Sampdoria\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eU.C. Sampdoria\u003c\/a\u003e, which is larger (~268K), look like this:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e5.1.3-1wm15\u003c\/td\u003e\u003ctd\u003e6.0.6-1wm1\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep10\u003c\/td\u003e\u003ctd\u003e2.3\u003c\/td\u003e\u003ctd\u003e2.3\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003e2.9\u003c\/td\u003e\u003ctd\u003e2.8\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep50\u003c\/td\u003e\u003ctd\u003e3.5\u003c\/td\u003e\u003ctd\u003e3.2\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003e4.3\u003c\/td\u003e\u003ctd\u003e3.8\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep90\u003c\/td\u003e\u003ctd\u003e5.7\u003c\/td\u003e\u003ctd\u003e4.9\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep95\u003c\/td\u003e\u003ctd\u003e8.5\u003c\/td\u003e\u003ctd\u003e6.0\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ep99\u003c\/td\u003e\u003ctd\u003e15.2\u003c\/td\u003e\u003ctd\u003e10.5\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eResults for \u003ctt class=\"remarkup-monospaced\"\u003eAccept-Encoding: gzip\u003c\/tt\u003e are similarly unremarkable.\u003c\/p\u003e\n\n\u003cp\u003eThe data does not seem to show significant differences between the performance of Varnish 5 and Varnish 6 when it comes to this sort of cache hits. As a next step we can try to observe the various Varnish milestone timestamps on miss\/pass traffic, and see if there's any difference there.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_496\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6535510\" id=\"6535510\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_495\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_161\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6535510\" data-sigil=\"has-tooltip\" data-meta=\"0_494\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 7:39 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 07:39:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_492\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_493\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks for trying that out. I ran the same command for much longer with higher concurrency. For Main_Page, I saw no difference.\u003c\/p\u003e\n\n\u003cp\u003eThen I tried \u003ca href=\"https:\/\/en.wikipedia.org\/wiki\/Barack_Obama\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eour go-to large article\u003c\/a\u003e, with the benchmark running concurrently on both hosts.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e.\/hey -n 1000000 -c 1000 -H "X-Forwarded-Proto: https" -host "en.wikipedia.org" http:\/\/localhost\/wiki\/Barack_Obama\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eA significant difference emerged. You can feel it right away seeing how long it takes to progress through the benchmark. The total runtime for cp3054 was 25% longer.\u003c\/p\u003e\n\n\u003cp\u003e\u003cstrong\u003ecp3052\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSummary:\n  Total:\t538.3917 secs\n  Slowest:\t3.6410 secs\n  Fastest:\t0.0098 secs\n  Average:\t0.5122 secs\n  Requests\/sec:\t1857.3839\n\nStatus code distribution:\n  [200]\t1000000 responses\n\nResponse time histogram:\n  0.010 [1]\t|\n  0.373 [408769]\t|\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\n  0.736 [380391]\t|\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\n  1.099 [141824]\t|\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\n  1.462 [40537]\t|\u220e\u220e\u220e\u220e\n  1.825 [17190]\t|\u220e\u220e\n  2.189 [8190]\t|\u220e\n  2.552 [1721]\t|\n  2.915 [527]\t|\n  3.278 [482]\t|\n  3.641 [368]\t|\n\nLatency distribution:\n  10% in 0.0878 secs\n  25% in 0.2483 secs\n  50% in 0.4448 secs\n  75% in 0.6846 secs\n  90% in 0.9694 secs\n  95% in 1.2226 secs\n  99% in 1.8680 secs\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003ecp3054\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSummary:\n  Total:\t676.5448 secs\n  Slowest:\t3.6392 secs\n  Fastest:\t0.0100 secs\n  Average:\t0.6690 secs\n  Requests\/sec:\t1478.0987\n\nStatus code distribution:\n  [200]\t1000000 responses\n\nResponse time histogram:\n  0.010 [1]\t|\n  0.373 [168978]\t|\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\n  0.736 [475582]\t|\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\n  1.099 [263823]\t|\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\u220e\n  1.462 [70445]\t|\u220e\u220e\u220e\u220e\u220e\u220e\n  1.825 [16243]\t|\u220e\n  2.187 [3935]\t|\n  2.550 [790]\t|\n  2.913 [166]\t|\n  3.276 [29]\t|\n  3.639 [8]\t|\n\nLatency distribution:\n  10% in 0.2956 secs\n  25% in 0.4492 secs\n  50% in 0.6256 secs\n  75% in 0.8387 secs\n  90% in 1.0774 secs\n  95% in 1.2484 secs\n  99% in 1.6502 secs\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eMaybe this issue only affects larger payloads?\u003c\/p\u003e\n\n\u003cp\u003eIs the Barack Obama article too big to be stored in Varnish? I can't remember what the limit is.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_505\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6535643\" id=\"6535643\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_504\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_163\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6535643\" data-sigil=\"has-tooltip\" data-meta=\"0_503\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 8:00 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 08:00:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_501\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_502\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_164\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've checked with curl and it does get cached by both hosts:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eX-Cache: cp3060 miss, cp3052 hit\/1000011\nX-Cache-Status: hit-front\n\nX-Cache: cp3060 miss, cp3054 hit\/1000019\nX-Cache-Status: hit-front\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eOddly, cp3054 has Content-Length defined in the headers I get back and not cp3052?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_514\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6535694\" id=\"6535694\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_513\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_165\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6535694\" data-sigil=\"has-tooltip\" data-meta=\"0_512\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 8:27 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 08:27:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_510\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_511\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_166\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6535510\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6535510\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThanks for trying that out. I ran the same command for much longer with higher concurrency. For Main_Page, I saw no difference.\u003c\/p\u003e\n\n\u003cp\u003eThen I tried \u003ca href=\"https:\/\/en.wikipedia.org\/wiki\/Barack_Obama\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eour go-to large article\u003c\/a\u003e, with the benchmark running concurrently on both hosts.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e.\/hey -n 1000000 -c 1000 -H "X-Forwarded-Proto: https" -host "en.wikipedia.org" http:\/\/localhost\/wiki\/Barack_Obama\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eOh, it seems that you've tried that between 07:25 and 07:35? During that timeframe, p75 webperf on cp3052 (Varnish 5) was bad, up to 4.3 seconds (!). See \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/EiAVq3FGz\/t264398?viewPanel=2&orgId=1&from=1602486144612&to=1602488403691\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethe usual dashboard\u003c\/a\u003e, but with the rate computed over 5 minutes instead of 1 hour:\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/okvwwgdouwqgjb6is3js\/PHID-FILE-xv64rrb2z3ajsytc6afb\/Screenshot_from_2020-10-12_09-48-30.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_60\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/ksuaxdt5o5i5qe4ednkb\/PHID-FILE-25k4c3fiuu5a7xf5lvz5\/preview-Screenshot_from_2020-10-12_09-48-30.png\" width=\"220\" height=\"123.75\" alt=\"Screenshot from 2020-10-12 09-48-30.png (1\u00d72 px, 162 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eI don't see the same effect on cp3054, interestingly. Also during the benchmark we've had a \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/wiU3SdEWk\/cache-host-drilldown?viewPanel=70&orgId=1&var-site=esams%20prometheus%2Fops&var-instance=cp3052&from=1602484675087&to=1602488770798\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebrief 502 spike\u003c\/a\u003e on cp3052, while that did not happen \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/wiU3SdEWk\/cache-host-drilldown?viewPanel=70&orgId=1&var-site=esams%20prometheus%2Fops&var-instance=cp3054&from=1602484675087&to=1602488770798\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eon cp3054\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cp\u003eThis might indicate a difference in the way v5 and v6 handle significant rates of concurrent requests with large responses. Can the work currently in progress on splitting webperf by cache_status (\u003ca href=\"\/T264987\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264987\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e) be extended to also include response size?\u003c\/p\u003e\n\n\u003cp\u003eWhile I do think the data is very interesting, we should move the benchmarks elsewhere to avoid disrupting production traffic (I'm not blaming Gilles for trying the command, just to be very clear). Perhaps deployment-cache-text06.deployment-prep.eqiad.wmflabs might be a good labs instance to use for this purpose, hoping that its limited resources do not end up interfering with the results.\u003c\/p\u003e\n\n\u003cp\u003eOn a different angle: while looking at various Varnish internal metrics, I've noticed that there's a very visible decrease in LRU lock operations on Varnish 6 compared to Varnish 5. See \u003ctt class=\"remarkup-monospaced\"\u003eincrease(varnish_lock_operations{instance=~"cp305[26]:9331",target="lru"}[5m])\u003c\/tt\u003e), the difference after the downgrade of cp3052 is quite clear:\u003cbr \/\u003e\n\u003cdiv class=\"phabricator-remarkup-embed-layout-left\"\u003e\u003ca href=\"https:\/\/phab.wmfusercontent.org\/file\/data\/h74rnluvsiq3m3a3u6yv\/PHID-FILE-dfix3svkvt7houdrr25b\/Screenshot_from_2020-10-12_10-12-52.png\" class=\"phabricator-remarkup-embed-image\" data-sigil=\"lightboxable\" data-meta=\"0_61\"\u003e\u003cimg src=\"https:\/\/phab.wmfusercontent.org\/file\/data\/erbub7b42ctvtakqzq4e\/PHID-FILE-4tojc3fuf2thk5ku52xk\/preview-Screenshot_from_2020-10-12_10-12-52.png\" width=\"220\" height=\"123.75\" alt=\"Screenshot from 2020-10-12 10-12-52.png (1\u00d72 px, 259 KB)\" \/\u003e\u003c\/a\u003e\u003c\/div\u003e\u003c\/p\u003e\n\n\u003cp\u003eGoing through the various commits that led to the 6.0.x series, it seems to me that the most likely reason is \u003ca href=\"https:\/\/github.com\/varnishcache\/varnish-cache\/issues\/2527\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/varnishcache\/varnish-cache\/issues\/2527\u003c\/a\u003e. If anything, less lock contention should result in \u003cstrong\u003ebetter\u003c\/strong\u003e performance on v6. However, as an experiment I've tried disabling \u003ctt class=\"remarkup-monospaced\"\u003elru_interval\u003c\/tt\u003e (ie: setting it to 0) on cp3054. Let's see what difference that makes performance-wise, if any. So far, I don't see anything interesting at all, so this is likely a red herring.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_524\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6535852\" id=\"6535852\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_523\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_167\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6535852\" data-sigil=\"has-tooltip\" data-meta=\"0_522\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 8:53 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 08:53:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_520\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_521\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_168\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI tried a bunch of different benchmark settings before I ended up using those, some of which definitely had too much concurrency and caused errors even on the benchmark responses themselves. So I'm not sure if you're seeing the failed attempts where the settings were really excessive or the final run on the Barack Obama article. It's probably worth running again. The fact that these latest settings led to 100% 200 responses contrary to the excessive ones leads me to believe that they're safe.\u003c\/p\u003e\n\n\u003cp\u003eMy shell history on those hosts isn't timestamped, so I can't confirm based on that.\u003c\/p\u003e\n\n\u003cp\u003eI can definitely add response size as another per-host dimension.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_534\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6535882\" id=\"6535882\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_533\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_169\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6535882\" data-sigil=\"has-tooltip\" data-meta=\"0_532\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 9:04 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 09:04:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_530\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_531\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_170\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIf it was indeed the last benchmark run, what you're saying is that cp3052 favored the benchmark requests over others, while cp3054 remained more fair in regards to the rest of the traffic? I think that even moving this to WMCS we're always going to run into problems like that with benchmarks. We're more likely to discover bias caused by the benchmark itself than track down the root cause of what we're seeing in organic traffic.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_543\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6536020\" id=\"6536020\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_542\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_171\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6536020\" data-sigil=\"has-tooltip\" data-meta=\"0_541\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 9:47 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 09:47:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_539\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_540\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_172\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6535643\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6535643\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI've checked with curl and it does get cached by both hosts:\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e[...]\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eOddly, cp3054 has Content-Length defined in the headers I get back and not cp3052?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eInteresting, which \u003ctt class=\"remarkup-monospaced\"\u003ecurl\u003c\/tt\u003e command exactly gives you no CL on 3052?\u003c\/p\u003e\n\n\u003cp\u003eI see CL being returned by Varnish on both hosts:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e09:43:05 ema@cp3054.esams.wmnet:~\n$ curl -v -H "X-Forwarded-Proto: https" -H "Host: en.wikipedia.org" localhost\/wiki\/Barack_Obama 2>&1 | grep Length\n< Content-Length: 1479288\n09:43:19 ema@cp3052.esams.wmnet:~\n$ curl -v -H "X-Forwarded-Proto: https" -H "Host: en.wikipedia.org" localhost\/wiki\/Barack_Obama 2>&1 | grep Length\n< Content-Length: 1479288\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_552\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6536163\" id=\"6536163\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_551\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_173\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6536163\" data-sigil=\"has-tooltip\" data-meta=\"0_550\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 10:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 10:22:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_548\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_549\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_174\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt was:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003egilles@cp3052:\/home\/ema$ curl -I -H "X-Forwarded-Proto: https" -H "Host: en.wikipedia.org" http:\/\/localhost\/wiki\/Barack_Obama | grep Content-Length\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\ngilles@cp3052:\/home\/ema$\n\ngilles@cp3054:\/home\/ema$ curl -I -H "X-Forwarded-Proto: https" -H "Host: en.wikipedia.org" http:\/\/localhost\/wiki\/Barack_Obama | grep Content-Length\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n  0 1444k    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\nContent-Length: 1479263\ngilles@cp3054:\/home\/ema$\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_562\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6536169\" id=\"6536169\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_561\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_175\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6536169\" data-sigil=\"has-tooltip\" data-meta=\"0_560\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 10:24 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 10:24:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_558\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_559\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_176\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI guess the difference is that I did a HEAD request. It's still reproducible right now.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_571\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6536565\" id=\"6536565\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_570\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_177\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6536565\" data-sigil=\"has-tooltip\" data-meta=\"0_569\"\u003e\u003cspan class=\"screen-only\"\u003eOct 12 2020, 12:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-12 12:39:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_567\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_568\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_178\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6536169\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_65\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6536169\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI guess the difference is that I did a HEAD request. It's still reproducible right now.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eOK, I've tried to see if other responses without body like 304s to conditional requests also showed any interesting difference, but they don't. So this might just be a change in interpretation of RFCs when it comes to HEAD, but nothing relevent for this task I think.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_581\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6538699\" id=\"6538699\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_580\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_179\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6538699\" data-sigil=\"has-tooltip\" data-meta=\"0_579\"\u003e\u003cspan class=\"screen-only\"\u003eOct 13 2020, 12:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-13 12:42:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_577\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_578\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've captured 30 minutes of data using varnishlog simultaneously on cp3052 and cp3054, using 4 variants of this command for hit-front, hit-local, miss and pass:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003evarnishlog -n frontend -q "RespHeader eq 'X-Cache-Status: hit-front' and ReqURL ~ '^\/wiki\/*'" -i Timestamp | grep Resp | awk '{print $5}' > cp3052-hit-front.log\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe above recorded the total response time from Varnish's perspective (duration shown on last Timestamp entry available for the transaction), for hit-front responses on \/wiki\/* URLs.\u003c\/p\u003e\n\n\u003cp\u003eThen I looked at percentiles on each dataset. The lower number is bolded.\u003c\/p\u003e\n\n\u003cp\u003e\u003cstrong\u003ehit-front\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e905671\u003c\/td\u003e\u003ctd\u003e0.000000\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000254\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.000291\u003c\/td\u003e\u003ctd\u003e0.001017\u003c\/td\u003e\u003ctd\u003e0.000336\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e5.420407\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.0305825\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e889287\u003c\/td\u003e\u003ctd\u003e0.000000\u003c\/td\u003e\u003ctd\u003e0.000256\u003c\/td\u003e\u003ctd\u003e0.000291\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000929\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000334\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e8.007157\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.0303075\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e0.8%\u003c\/td\u003e\u003ctd\u003e0%\u003c\/td\u003e\u003ctd\u003e-9.5%\u003c\/td\u003e\u003ctd\u003e-0.59%\u003c\/td\u003e\u003ctd\u003e47%\u003c\/td\u003e\u003ctd\u003e-0.9%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003ehit-local\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e20480\u003c\/td\u003e\u003ctd\u003e0.000611\u003c\/td\u003e\u003ctd\u003e0.003231\u003c\/td\u003e\u003ctd\u003e0.006207\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.036319\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.050616\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e1.945326\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.07614508\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e24120\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000602\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.002979\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.005785\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.037372\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.041039\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e1.973892\u003c\/td\u003e\u003ctd\u003e0.08716184\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e-1.5%\u003c\/td\u003e\u003ctd\u003e-8.5%\u003c\/td\u003e\u003ctd\u003e-7.3%\u003c\/td\u003e\u003ctd\u003e2.9%\u003c\/td\u003e\u003ctd\u003e-23.3 %\u003c\/td\u003e\u003ctd\u003e1.5%\u003c\/td\u003e\u003ctd\u003e14.5%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003emiss\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e243029\u003c\/td\u003e\u003ctd\u003e0.1162\u003c\/td\u003e\u003ctd\u003e0.3125\u003c\/td\u003e\u003ctd\u003e0.4136\u003c\/td\u003e\u003ctd\u003e0.4876\u003c\/td\u003e\u003ctd\u003e0.5500\u003c\/td\u003e\u003ctd\u003e7.2766\u003c\/td\u003e\u003ctd\u003e0.319049\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e251676\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.1161\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.2954\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.4033\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.4749\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.5459\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e4.9899\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.2933685\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e-0.08%\u003c\/td\u003e\u003ctd\u003e-5.8%\u003c\/td\u003e\u003ctd\u003e-2.5%\u003c\/td\u003e\u003ctd\u003e-2.7%\u003c\/td\u003e\u003ctd\u003e-0.7%\u003c\/td\u003e\u003ctd\u003e-45.8%\u003c\/td\u003e\u003ctd\u003e-10.9%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003epass\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e67811\u003c\/td\u003e\u003ctd\u003e0.1164\u003c\/td\u003e\u003ctd\u003e0.1914\u003c\/td\u003e\u003ctd\u003e0.3325\u003c\/td\u003e\u003ctd\u003e0.4667\u003c\/td\u003e\u003ctd\u003e0.5523\u003c\/td\u003e\u003ctd\u003e7.4085\u003c\/td\u003e\u003ctd\u003e0.5117037\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e64170\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.1159\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.1784\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.2968\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.4295\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.5392\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e4.9915\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.3971842\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e-0.4%\u003c\/td\u003e\u003ctd\u003e-7.3%\u003c\/td\u003e\u003ctd\u003e-12%\u003c\/td\u003e\u003ctd\u003e-8.7%\u003c\/td\u003e\u003ctd\u003e-2.4%\u003c\/td\u003e\u003ctd\u003e-48.4%\u003c\/td\u003e\u003ctd\u003e-28.8%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eI don't know if the time reported by Varnish includes the transmission time to the client, but as far as its own telemetry goes, cp3054 looks faster or at least as performant.\u003c\/p\u003e\n\n\u003cp\u003eSeeing this, I pulled the RUM data for cp3052 and cp3054 for some offline processing, using this HQL query that reproduces the filtering we do when generating the Prometheus metric that ends up on \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/M7xQ_BeWk\/response-time-by-host\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ethe per-host dashboard\u003c\/a\u003e. It looks at data from 2020-10-07 onwards:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSELECT event.responsestart - event.fetchstart FROM event.navigationtiming WHERE year = 2020 AND month = 10 AND day > 6 AND recvfrom = 'cp3052.esams.wmnet' AND event.isOversample = false AND event.responsestart - event.fetchstart >= 0;\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e128452\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e108\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e257\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e469\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e520\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e148264\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e1343\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e128176\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e115\u003c\/td\u003e\u003ctd\u003e285\u003c\/td\u003e\u003ctd\u003e483\u003c\/td\u003e\u003ctd\u003e543\u003c\/td\u003e\u003ctd\u003e625664\u003c\/td\u003e\u003ctd\u003e2129\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e0.9%\u003c\/td\u003e\u003ctd\u003e10.9%\u003c\/td\u003e\u003ctd\u003e2.9%\u003c\/td\u003e\u003ctd\u003e4.4%\u003c\/td\u003e\u003ctd\u003e333%\u003c\/td\u003e\u003ctd\u003e58.5%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eI've also looked at the same data, this time including the \u003ca href=\"https:\/\/github.com\/wikimedia\/operations-mediawiki-config\/blob\/master\/wmf-config\/InitialiseSettings.php#L22378-L22390\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eoversamples\u003c\/a\u003e. That extra data isn't included in the Prometheus per-host metrics.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e670108\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e178\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e403\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e597\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e650\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e245149\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e1392\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e674341\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e189\u003c\/td\u003e\u003ctd\u003e420\u003c\/td\u003e\u003ctd\u003e604\u003c\/td\u003e\u003ctd\u003e660\u003c\/td\u003e\u003ctd\u003e997972\u003c\/td\u003e\u003ctd\u003e2184\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e\u003c\/td\u003e\u003ctd\u003e6.2%\u003c\/td\u003e\u003ctd\u003e4.2%\u003c\/td\u003e\u003ctd\u003e1.2%\u003c\/td\u003e\u003ctd\u003e1.5%\u003c\/td\u003e\u003ctd\u003e307%\u003c\/td\u003e\u003ctd\u003e56.9%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003ePercentiles are a little higher for cp3054, less pronounced when including the \u003ca href=\"https:\/\/github.com\/wikimedia\/operations-mediawiki-config\/blob\/master\/wmf-config\/InitialiseSettings.php#L22378-L22390\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003egeographically-biased oversamples\u003c\/a\u003e. The maximum value and standard deviation are a lot higher in both cases.\u003c\/p\u003e\n\n\u003cp\u003eIt's possible that the extra time comes from something Varnish doesn't measure. It's unclear to me whether the last Timestamp in a Varnish response includes the time it took to actually ship the bytes to the client (ats-tls in this case?) and have them acknowledged.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_590\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6563442\" id=\"6563442\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_589\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_181\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6563442\" data-sigil=\"has-tooltip\" data-meta=\"0_588\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 10:06 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 10:06:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_586\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_587\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_182\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6538699\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6538699\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI've captured 30 minutes of data using varnishlog simultaneously on cp3052 and cp3054, using 4 variants of this command for hit-front, hit-local, miss and pass:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003evarnishlog -n frontend -q "RespHeader eq 'X-Cache-Status: hit-front'" -q "ReqURL ~ '^\/wiki\/*'" -i Timestamp | grep Resp | awk '{print $5}' > cp3052-hit-front.log\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003e[...]\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIt's possible that the extra time comes from something Varnish doesn't measure. It's unclear to me whether the last Timestamp in a Varnish response includes the time it took to actually ship the bytes to the client (ats-tls in this case?) and have them acknowledged.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThe command above prints the 5th field of the \u003ctt class=\"remarkup-monospaced\"\u003eResp\u003c\/tt\u003e timestamp as reported by Varnish. From \u003ctt class=\"remarkup-monospaced\"\u003evsl(7)\u003c\/tt\u003e:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eThe reported fields show the absolute time of the event,  the  time\nspent  since  the  start of the task and the time spent since\nthe last timestamp was logged.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eLog lines look like this:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e-   Timestamp      Resp: 1603186764.780741 0.000476 0.000081\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd Resp is:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eResp   Delivery of response to the client finished.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo that would be the time spent between the start of the task and the full delivery of the response to the client including the whole body. The client is ats-tls for https traffic, the actual user-agent for http.\u003c\/p\u003e\n\n\u003cp\u003eMy understanding is that the RUM data instead shows the time spent between the beginning of the client request, \u003ctt class=\"remarkup-monospaced\"\u003eevent.fetchstart\u003c\/tt\u003e, and the moment when the first bytes are received, \u003ctt class=\"remarkup-monospaced\"\u003eevent.responsestart\u003c\/tt\u003e. It seems to me that if we want to look into Varnish's own telemetry and compare things with \u003ctt class=\"remarkup-monospaced\"\u003eevent.responsestart - event.fetchstart\u003c\/tt\u003e, then we should use the so called \u003ctt class=\"remarkup-monospaced\"\u003eProcess\u003c\/tt\u003e timestamp:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eProcess\n       Processing  finished,  ready to deliver the client re\u2010\n       sponse.\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_599\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6564065\" id=\"6564065\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_598\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_183\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6564065\" data-sigil=\"has-tooltip\" data-meta=\"0_597\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 1:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 13:08:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_595\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_596\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_184\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6538699\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6538699\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSELECT event.responsestart - event.fetchstart FROM event.navigationtiming WHERE year = 2020 AND month = 10 AND day > 6 AND recvfrom = 'cp3052.esams.wmnet' AND event.isOversample = false AND event.responsestart - event.fetchstart >= 0;\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI've noticed that on nodes with Varnish 6 the worst \u003ctt class=\"remarkup-monospaced\"\u003etime_firstbyte\u003c\/tt\u003e values reported by ats-tls are very often around 26 seconds, and they're due to etherpad. Can you try this once again, but excluding \u003ctt class=\"remarkup-monospaced\"\u003eHost: etherpad.wikimedia.org\u003c\/tt\u003e?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_608\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6564285\" id=\"6564285\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_607\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_185\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6564285\" data-sigil=\"has-tooltip\" data-meta=\"0_606\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 1:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 13:37:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_604\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_605\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_186\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 635298 had a related patch set uploaded (by Ema; owner: Ema):\u003cbr \/\u003e\n[operations\/puppet@production] varnish: fix websockets on 6.x\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/635298\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/635298\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_611\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6564286\" id=\"6564286\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_610\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_187\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_188\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6564286\" data-sigil=\"has-tooltip\" data-meta=\"0_609\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 1:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 13:37:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_620\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6564303\" id=\"6564303\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_619\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_189\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6564303\" data-sigil=\"has-tooltip\" data-meta=\"0_618\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 1:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 13:41:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_616\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_617\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_190\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6564065\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6564065\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ema\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@ema\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T264398#6538699\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT264398#6538699\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSELECT event.responsestart - event.fetchstart FROM event.navigationtiming WHERE year = 2020 AND month = 10 AND day > 6 AND recvfrom = 'cp3052.esams.wmnet' AND event.isOversample = false AND event.responsestart - event.fetchstart >= 0;\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI've noticed that on nodes with Varnish 6 the worst \u003ctt class=\"remarkup-monospaced\"\u003etime_firstbyte\u003c\/tt\u003e values reported by ats-tls are very often around 26 seconds, and they're due to etherpad. Can you try this once again, but excluding \u003ctt class=\"remarkup-monospaced\"\u003eHost: etherpad.wikimedia.org\u003c\/tt\u003e?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWe don't collect RUM data on etherpad.wikimedia.org. As for Varnish data, it checks for ReqURL ~ '^\/wiki\/*. As far as I know, etherpad URLs all start with \/p\/.\u003c\/p\u003e\n\n\u003cp\u003eFair point about TTFB from Varnish's perspective, I'll re-run the collection of data I did before, this time on \u003ctt class=\"remarkup-monospaced\"\u003eProcess\u003c\/tt\u003e instead of \u003ctt class=\"remarkup-monospaced\"\u003eResp\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_629\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6564517\" id=\"6564517\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_628\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6564517\" data-sigil=\"has-tooltip\" data-meta=\"0_627\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 2:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 14:33:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_625\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_626\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_192\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 635298 \u003cstrong\u003emerged\u003c\/strong\u003e by Ema:\u003cbr \/\u003e\n[operations\/puppet@production] varnish: fix websockets on 6.x\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/635298\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/635298\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_632\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/i4r43smu5b7mse27ryv6\/PHID-FILE-6psycbhbwmt2lokqybwm\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Maintenance_bot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6564701\" id=\"6564701\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_631\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Maintenance_bot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_193\"\u003eMaintenance_bot\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6564701\" data-sigil=\"has-tooltip\" data-meta=\"0_630\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 3:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 15:10:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_642\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6564734\" id=\"6564734\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_641\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_195\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#6564734\" data-sigil=\"has-tooltip\" data-meta=\"0_640\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 3:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 15:23:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_638\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_639\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHere's the same data collected with commands like the following (using \u003ctt class=\"remarkup-monospaced\"\u003eProcess\u003c\/tt\u003e), over a 30 minute period.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003evarnishlog -n frontend -q "RespHeader eq 'X-Cache-Status: hit-front' and ReqURL ~ '^\/wiki\/*'" -i Timestamp | grep Process | awk '{print $5}' > cp3052-hit-front.log\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003ehit-front\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e1175096\u003c\/td\u003e\u003ctd\u003e0.000138\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000216\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000247\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000697\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000278\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e4.269078\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.01909279\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e1133226\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000000\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.000221\u003c\/td\u003e\u003ctd\u003e0.000249\u003c\/td\u003e\u003ctd\u003e0.001107\u003c\/td\u003e\u003ctd\u003e0.000279\u003c\/td\u003e\u003ctd\u003e11.836655\u003c\/td\u003e\u003ctd\u003e0.03249044\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e-3.7%\u003c\/td\u003e\u003ctd\u003e-100%\u003c\/td\u003e\u003ctd\u003e2.3%\u003c\/td\u003e\u003ctd\u003e0.8%\u003c\/td\u003e\u003ctd\u003e58.8%\u003c\/td\u003e\u003ctd\u003e0.4%\u003c\/td\u003e\u003ctd\u003e177.2%\u003c\/td\u003e\u003ctd\u003e70.2%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003ehit-local\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e32768\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.000549\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.001042\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.001237\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.014784\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.001808\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e1.724109\u003c\/td\u003e\u003ctd\u003e0.05559903\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e32312\u003c\/td\u003e\u003ctd\u003e0.000581\u003c\/td\u003e\u003ctd\u003e0.001074\u003c\/td\u003e\u003ctd\u003e0.001290\u003c\/td\u003e\u003ctd\u003e0.015297\u003c\/td\u003e\u003ctd\u003e0.001834\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e1.555266\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.0528903\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e-1.4%\u003c\/td\u003e\u003ctd\u003e5.8%\u003c\/td\u003e\u003ctd\u003e3.1%\u003c\/td\u003e\u003ctd\u003e4.3%\u003c\/td\u003e\u003ctd\u003e3.7%\u003c\/td\u003e\u003ctd\u003e1.4%\u003c\/td\u003e\u003ctd\u003e-10.8%\u003c\/td\u003e\u003ctd\u003e-5.1%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003emiss\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e282624\u003c\/td\u003e\u003ctd\u003e0.1162\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.2816\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.3493\u003c\/td\u003e\u003ctd\u003e0.4139\u003c\/td\u003e\u003ctd\u003e0.4357\u003c\/td\u003e\u003ctd\u003e7.0794\u003c\/td\u003e\u003ctd\u003e0.2720957\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e329045\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.1161\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e0.2829\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.3450\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.4053\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.4297\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e4.2102\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.2469989\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e16.4%\u003c\/td\u003e\u003ctd\u003e-0.1%\u003c\/td\u003e\u003ctd\u003e0.5%\u003c\/td\u003e\u003ctd\u003e-1.2%\u003c\/td\u003e\u003ctd\u003e-2.1%\u003c\/td\u003e\u003ctd\u003e-1.4%\u003c\/td\u003e\u003ctd\u003e-68.1%\u003c\/td\u003e\u003ctd\u003e-10.16%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003epass\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e83285\u003c\/td\u003e\u003ctd\u003e0.1160\u003c\/td\u003e\u003ctd\u003e0.2023\u003c\/td\u003e\u003ctd\u003e0.3160\u003c\/td\u003e\u003ctd\u003e0.3917\u003c\/td\u003e\u003ctd\u003e0.4350\u003c\/td\u003e\u003ctd\u003e5.7782\u003c\/td\u003e\u003ctd\u003e0.3779185\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e99214\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.0000\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.1815\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.3058\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.3630\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.4315\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e4.6277\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e0.2776744\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003e19.1%\u003c\/td\u003e\u003ctd\u003e-100%\u003c\/td\u003e\u003ctd\u003e-11.4%\u003c\/td\u003e\u003ctd\u003e-3.3%\u003c\/td\u003e\u003ctd\u003e-7.9%\u003c\/td\u003e\u003ctd\u003e-0.8%\u003c\/td\u003e\u003ctd\u003e-24.8%\u003c\/td\u003e\u003ctd\u003e-36.1%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003ecp3054 seems to be consistently a little faster for miss and pass, and overall a little slower for hit-front and hit-local.  But I still can't see anything in the order of the extra tens of milliseconds seen on clients.\u003c\/p\u003e\n\n\u003cp\u003eI thought I could also look at RUM data all the way to responseEnd, which isn't information we have per host on Grafana. This can show us whether the higher TTFB might be compensated by subsequent faster delivery of data.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eSELECT event.responseend - event.fetchstart FROM event.navigationtiming WHERE year = 2020 AND month = 10 AND day > 6 AND recvfrom = 'cp3052.esams.wmnet' AND event.isOversample = false AND event.responseend - event.fetchstart >= 0;\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThere were a few freak high values that I filtered out (I removed anything greater than 10,000,000. i.e. 2.7 hours...)\u003c\/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eWithout geo oversamples\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e276804\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e128\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e306\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e589\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e618\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e7610298\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e14665\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e277133\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e146\u003c\/td\u003e\u003ctd\u003e335\u003c\/td\u003e\u003ctd\u003e621\u003c\/td\u003e\u003ctd\u003e642\u003c\/td\u003e\u003ctd\u003e7866575\u003c\/td\u003e\u003ctd\u003e16909\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003en\/a\u003c\/td\u003e\u003ctd\u003e0%\u003c\/td\u003e\u003ctd\u003e14%\u003c\/td\u003e\u003ctd\u003e9.5%\u003c\/td\u003e\u003ctd\u003e5.4%\u003c\/td\u003e\u003ctd\u003e3.9%\u003c\/td\u003e\u003ctd\u003e3.4%\u003c\/td\u003e\u003ctd\u003e15.3%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003e\u003cstrong\u003eWith geo oversamples\u003c\/strong\u003e\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-table-wrap\"\u003e\u003ctable class=\"remarkup-table\"\u003e\n\u003ctr\u003e\u003ctd\u003ehost\u003c\/td\u003e\u003ctd\u003esamples\u003c\/td\u003e\u003ctd\u003emin\u003c\/td\u003e\u003ctd\u003ep25\u003c\/td\u003e\u003ctd\u003emedian\u003c\/td\u003e\u003ctd\u003emean\u003c\/td\u003e\u003ctd\u003ep75\u003c\/td\u003e\u003ctd\u003emax\u003c\/td\u003e\u003ctd\u003estddev\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3052\u003c\/td\u003e\u003ctd\u003e1464285\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e210\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e461\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e698\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e767\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e7610298\u003c\/strong\u003e\u003c\/td\u003e\u003ctd\u003e\u003cstrong\u003e6939\u003c\/strong\u003e\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003ecp3054\u003c\/td\u003e\u003ctd\u003e1466906\u003c\/td\u003e\u003ctd\u003e0\u003c\/td\u003e\u003ctd\u003e222\u003c\/td\u003e\u003ctd\u003e483\u003c\/td\u003e\u003ctd\u003e713\u003c\/td\u003e\u003ctd\u003e778\u003c\/td\u003e\u003ctd\u003e7866575\u003c\/td\u003e\u003ctd\u003e8016\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDifference\u003c\/td\u003e\u003ctd\u003en\/a\u003c\/td\u003e\u003ctd\u003e0%\u003c\/td\u003e\u003ctd\u003e5.7%\u003c\/td\u003e\u003ctd\u003e4.8%\u003c\/td\u003e\u003ctd\u003e2.1%\u003c\/td\u003e\u003ctd\u003e1.4%\u003c\/td\u003e\u003ctd\u003e3.4%\u003c\/td\u003e\u003ctd\u003e15.5%\u003c\/td\u003e\u003c\/tr\u003e\n\u003ctr\u003e\u003c\/tr\u003e\n\u003c\/table\u003e\u003c\/div\u003e\n\n\u003cp\u003eIt doesn't seem like data is delivered faster, as the difference remains all the way to \u003ctt class=\"remarkup-monospaced\"\u003eresponseend\u003c\/tt\u003e.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_651\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/oczrgb2xczu3hebwulsk\/PHID-FILE-3raolhvprjakga3u36ek\/alphanumeric_lato-white_B.png-_5880b0-255%2C255%2C255%2C0.7.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/BBlack\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6565046\" id=\"6565046\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_650\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_197\"\u003eBBlack\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6565046\" data-sigil=\"has-tooltip\" data-meta=\"0_649\"\u003e\u003cspan class=\"screen-only\"\u003eOct 20 2020, 4:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-20 16:28:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_647\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_648\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI stumbled on \u003ca href=\"\/T266040\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT266040\u003c\/span\u003e\u003c\/a\u003e while looking at something unrelated, but now I'm remembering that earlier in this ticket, there was some mention of a possible correlation with larger response sizes here in this report as well.  The stuff in \u003ca href=\"\/T266040\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT266040\u003c\/span\u003e\u003c\/a\u003e would have some statistical negative effect primarily on objects >= 256KB (the hieradata-controlled \u003ctt class=\"remarkup-monospaced\"\u003elarge_objects_cutoff\u003c\/tt\u003e), and a secondary effect on overall backend caching efficiency for everything else (by wasting space on pointless duplications of content).  AFAIK what we're looking at in that ticket isn't something that changed from V5 to V6, though... but I could be wrong, or it could be that some other subtle change in V6 behavior exacerbated the impact.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},[],[],{"hovercardSpec":{"objectPHID":"PHID-TASK-iru2n5ru37cnb7wluhqm"}},{"phid":"PHID-FILE-nscklonegls555v3j4ah","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/2zoibklah237mmpudvsu\/PHID-FILE-nscklonegls555v3j4ah\/Screenshot_2020-10-02_at_10.56.42.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/2zoibklah237mmpudvsu\/PHID-FILE-nscklonegls555v3j4ah\/Screenshot_2020-10-02_at_10.56.42.png","alt":"Screenshot 2020-10-02 at 10.56.42.png (534\u00d71 px, 160 KB)","viewable":true,"monogram":"F32370865"},{"phid":"PHID-FILE-gp2s5oeb6nzwlqttfrzy","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/voy6rpw7bw6bzphuejpg\/PHID-FILE-gp2s5oeb6nzwlqttfrzy\/Screenshot_2020-10-02_at_10.57.35.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/voy6rpw7bw6bzphuejpg\/PHID-FILE-gp2s5oeb6nzwlqttfrzy\/Screenshot_2020-10-02_at_10.57.35.png","alt":"Screenshot 2020-10-02 at 10.57.35.png (550\u00d71 px, 150 KB)","viewable":true,"monogram":"F32370869"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3q5h3y4iyeinkcl22nwv"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"phid":"PHID-FILE-ujz6szx4jylz7vvjeur7","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/3f37eru3n4nrbpslx7ld\/PHID-FILE-ujz6szx4jylz7vvjeur7\/Screenshot_2020-10-05_at_10.08.52.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/3f37eru3n4nrbpslx7ld\/PHID-FILE-ujz6szx4jylz7vvjeur7\/Screenshot_2020-10-05_at_10.08.52.png","alt":"Screenshot 2020-10-05 at 10.08.52.png (426\u00d71 px, 77 KB)","viewable":true,"monogram":"F32373885"},{"phid":"PHID-FILE-xa7pwmuaoadzq2kzu4d2","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/fmomlq4w6pzu4mhlafoc\/PHID-FILE-xa7pwmuaoadzq2kzu4d2\/Screenshot_2020-10-05_at_10.08.10.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/fmomlq4w6pzu4mhlafoc\/PHID-FILE-xa7pwmuaoadzq2kzu4d2\/Screenshot_2020-10-05_at_10.08.10.png","alt":"Screenshot 2020-10-05 at 10.08.10.png (430\u00d71 px, 67 KB)","viewable":true,"monogram":"F32373886"},{"phid":"PHID-FILE-rbes2l3pv7uaoozdxvgf","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/cvn4spewccugtvpga5xi\/PHID-FILE-rbes2l3pv7uaoozdxvgf\/Screenshot_2020-10-05_at_10.07.56.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/cvn4spewccugtvpga5xi\/PHID-FILE-rbes2l3pv7uaoozdxvgf\/Screenshot_2020-10-05_at_10.07.56.png","alt":"Screenshot 2020-10-05 at 10.07.56.png (437\u00d71 px, 78 KB)","viewable":true,"monogram":"F32373887"},{"phid":"PHID-FILE-sov7fk4zhmin64nhip4x","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/4wjbiwu2muo6m6hqliub\/PHID-FILE-sov7fk4zhmin64nhip4x\/Screenshot_2020-10-05_at_10.08.26.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/4wjbiwu2muo6m6hqliub\/PHID-FILE-sov7fk4zhmin64nhip4x\/Screenshot_2020-10-05_at_10.08.26.png","alt":"Screenshot 2020-10-05 at 10.08.26.png (428\u00d71 px, 79 KB)","viewable":true,"monogram":"F32373888"},{"phid":"PHID-FILE-bmhkh36mnihdtyxywyzm","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/ajn4lyyx2itnuedhlptk\/PHID-FILE-bmhkh36mnihdtyxywyzm\/Screenshot_2020-10-05_at_10.08.34.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/ajn4lyyx2itnuedhlptk\/PHID-FILE-bmhkh36mnihdtyxywyzm\/Screenshot_2020-10-05_at_10.08.34.png","alt":"Screenshot 2020-10-05 at 10.08.34.png (430\u00d71 px, 78 KB)","viewable":true,"monogram":"F32373890"},{"phid":"PHID-FILE-ftl6cpzyetcgrvsxtxox","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/l4khcod6ftntvtlsf4o6\/PHID-FILE-ftl6cpzyetcgrvsxtxox\/Screenshot_2020-10-05_at_10.08.42.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/l4khcod6ftntvtlsf4o6\/PHID-FILE-ftl6cpzyetcgrvsxtxox\/Screenshot_2020-10-05_at_10.08.42.png","alt":"Screenshot 2020-10-05 at 10.08.42.png (428\u00d71 px, 77 KB)","viewable":true,"monogram":"F32373889"},{"phid":"PHID-FILE-2cmag5mkyj6bzzqcjyfl","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/uziwspgicriwv3gkn2zw\/PHID-FILE-2cmag5mkyj6bzzqcjyfl\/Screenshot_2020-10-05_at_10.08.03.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/uziwspgicriwv3gkn2zw\/PHID-FILE-2cmag5mkyj6bzzqcjyfl\/Screenshot_2020-10-05_at_10.08.03.png","alt":"Screenshot 2020-10-05 at 10.08.03.png (432\u00d71 px, 78 KB)","viewable":true,"monogram":"F32373891"},{"phid":"PHID-FILE-cj7moxk4buoh5pmjdszv","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/slp774wqvvhcodh3nljd\/PHID-FILE-cj7moxk4buoh5pmjdszv\/Screenshot_2020-10-05_at_10.08.18.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/slp774wqvvhcodh3nljd\/PHID-FILE-cj7moxk4buoh5pmjdszv\/Screenshot_2020-10-05_at_10.08.18.png","alt":"Screenshot 2020-10-05 at 10.08.18.png (426\u00d71 px, 77 KB)","viewable":true,"monogram":"F32373892"},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"phid":"PHID-FILE-ej7ib7xjjouo7oismsg3","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/v37p5kgxunrjf6lwtmq6\/PHID-FILE-ej7ib7xjjouo7oismsg3\/Screenshot_from_2020-10-05_12-50-09.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/v37p5kgxunrjf6lwtmq6\/PHID-FILE-ej7ib7xjjouo7oismsg3\/Screenshot_from_2020-10-05_12-50-09.png","alt":"Screenshot from 2020-10-05 12-50-09.png (1\u00d72 px, 383 KB)","viewable":true,"monogram":"F32374208"},{"phid":"PHID-FILE-hzhq4ge43l4bqrmc3y5s","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/5q5qo252udtb52bq545f\/PHID-FILE-hzhq4ge43l4bqrmc3y5s\/Screenshot_2020-10-06_at_15.08.11.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/5q5qo252udtb52bq545f\/PHID-FILE-hzhq4ge43l4bqrmc3y5s\/Screenshot_2020-10-06_at_15.08.11.png","alt":"Screenshot 2020-10-06 at 15.08.11.png (533\u00d71 px, 88 KB)","viewable":true,"monogram":"F32375693"},{"hovercardSpec":{"objectPHID":"PHID-TASK-tnlopjibqyn2cz37mxe7"}},{"phid":"PHID-FILE-gnj6nje3kcaleavb7oqo","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/hmfabls54bbpsket25pp\/PHID-FILE-gnj6nje3kcaleavb7oqo\/Screenshot_from_2020-10-07_10-41-40.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/hmfabls54bbpsket25pp\/PHID-FILE-gnj6nje3kcaleavb7oqo\/Screenshot_from_2020-10-07_10-41-40.png","alt":"Screenshot from 2020-10-07 10-41-40.png (1\u00d72 px, 229 KB)","viewable":true,"monogram":"F32376629"},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-ctafnlifn64axvokqjy3"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-obttcll32ornyejbwu53"}},{"phid":"PHID-FILE-rxqsvkfoxriytd2febjz","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/7pkyghkhf2tlvqf4eeeb\/PHID-FILE-rxqsvkfoxriytd2febjz\/Screenshot_2020-10-07_at_12.09.54.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/7pkyghkhf2tlvqf4eeeb\/PHID-FILE-rxqsvkfoxriytd2febjz\/Screenshot_2020-10-07_at_12.09.54.png","alt":"Screenshot 2020-10-07 at 12.09.54.png (971\u00d71 px, 143 KB)","viewable":true,"monogram":"F32376688"},{"phid":"PHID-FILE-moxp2txy3uyup3zvhizv","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/zqrpzgzxdlmsp2u6hbsx\/PHID-FILE-moxp2txy3uyup3zvhizv\/Screenshot_2020-10-07_at_12.10.47.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/zqrpzgzxdlmsp2u6hbsx\/PHID-FILE-moxp2txy3uyup3zvhizv\/Screenshot_2020-10-07_at_12.10.47.png","alt":"Screenshot 2020-10-07 at 12.10.47.png (578\u00d71 px, 69 KB)","viewable":true,"monogram":"F32376691"},{"phid":"PHID-FILE-4x4qmlvg5micc75h7hkv","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/ujyohhaeke4nlgzhu43w\/PHID-FILE-4x4qmlvg5micc75h7hkv\/Screenshot_2020-10-07_at_12.11.53.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/ujyohhaeke4nlgzhu43w\/PHID-FILE-4x4qmlvg5micc75h7hkv\/Screenshot_2020-10-07_at_12.11.53.png","alt":"Screenshot 2020-10-07 at 12.11.53.png (581\u00d71 px, 70 KB)","viewable":true,"monogram":"F32376693"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-ctafnlifn64axvokqjy3"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-ctafnlifn64axvokqjy3"}},{"phid":"PHID-FILE-vvsmwngpykn3gxkhtju6","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/6nhezxtqndshu5qu3gq3\/PHID-FILE-vvsmwngpykn3gxkhtju6\/Screenshot_2020-10-08_at_08.44.41.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/6nhezxtqndshu5qu3gq3\/PHID-FILE-vvsmwngpykn3gxkhtju6\/Screenshot_2020-10-08_at_08.44.41.png","alt":"Screenshot 2020-10-08 at 08.44.41.png (345\u00d71 px, 60 KB)","viewable":true,"monogram":"F32377846"},{"phid":"PHID-FILE-pzttkzxf5z7kji4d5a6v","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/xu2ahlmkyrq22hxaa5nn\/PHID-FILE-pzttkzxf5z7kji4d5a6v\/Screenshot_2020-10-08_at_08.45.00.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/xu2ahlmkyrq22hxaa5nn\/PHID-FILE-pzttkzxf5z7kji4d5a6v\/Screenshot_2020-10-08_at_08.45.00.png","alt":"Screenshot 2020-10-08 at 08.45.00.png (342\u00d71 px, 62 KB)","viewable":true,"monogram":"F32377848"},{"phid":"PHID-FILE-ltoeng6hlgfiliavfmzn","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/mapnzqw2epnn7tykxlnk\/PHID-FILE-ltoeng6hlgfiliavfmzn\/Screenshot_2020-10-08_at_08.45.10.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/mapnzqw2epnn7tykxlnk\/PHID-FILE-ltoeng6hlgfiliavfmzn\/Screenshot_2020-10-08_at_08.45.10.png","alt":"Screenshot 2020-10-08 at 08.45.10.png (345\u00d71 px, 53 KB)","viewable":true,"monogram":"F32377847"},{"phid":"PHID-FILE-5udkcftvxkt2brxzxhjw","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/bklwp4yhia6b7emgeslo\/PHID-FILE-5udkcftvxkt2brxzxhjw\/Screenshot_2020-10-08_at_08.44.53.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/bklwp4yhia6b7emgeslo\/PHID-FILE-5udkcftvxkt2brxzxhjw\/Screenshot_2020-10-08_at_08.44.53.png","alt":"Screenshot 2020-10-08 at 08.44.53.png (344\u00d71 px, 60 KB)","viewable":true,"monogram":"F32377849"},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-tut4u6p2planh6pamhqo"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3q5h3y4iyeinkcl22nwv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-tut4u6p2planh6pamhqo"}},{"phid":"PHID-FILE-ovn76whgqfzphooz7lfq","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/yuykvqyhwkb6wmwnuzvm\/PHID-FILE-ovn76whgqfzphooz7lfq\/Screenshot_2020-10-09_at_09.46.33.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/yuykvqyhwkb6wmwnuzvm\/PHID-FILE-ovn76whgqfzphooz7lfq\/Screenshot_2020-10-09_at_09.46.33.png","alt":"Screenshot 2020-10-09 at 09.46.33.png (267\u00d7533 px, 45 KB)","viewable":true,"monogram":"F32379125"},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-tut4u6p2planh6pamhqo"}},{"phid":"PHID-FILE-xv64rrb2z3ajsytc6afb","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/okvwwgdouwqgjb6is3js\/PHID-FILE-xv64rrb2z3ajsytc6afb\/Screenshot_from_2020-10-12_09-48-30.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/okvwwgdouwqgjb6is3js\/PHID-FILE-xv64rrb2z3ajsytc6afb\/Screenshot_from_2020-10-12_09-48-30.png","alt":"Screenshot from 2020-10-12 09-48-30.png (1\u00d72 px, 162 KB)","viewable":true,"monogram":"F32381895"},{"phid":"PHID-FILE-dfix3svkvt7houdrr25b","uri":"https:\/\/phab.wmfusercontent.org\/file\/data\/h74rnluvsiq3m3a3u6yv\/PHID-FILE-dfix3svkvt7houdrr25b\/Screenshot_from_2020-10-12_10-12-52.png","dUri":"https:\/\/phab.wmfusercontent.org\/file\/download\/h74rnluvsiq3m3a3u6yv\/PHID-FILE-dfix3svkvt7houdrr25b\/Screenshot_from_2020-10-12_10-12-52.png","alt":"Screenshot from 2020-10-12 10-12-52.png (1\u00d72 px, 259 KB)","viewable":true,"monogram":"F32381905"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev","contextPHID":"PHID-TASK-vz4rgcojxisv7jwqnk57"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-j6grynzibh4q2xsg7the"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-j6grynzibh4q2xsg7the"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-iidqqm4umcxxuld"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-l4q7np7bdbbq4kz"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-gvtmio6mk2px2w5"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-uwfs3sfbbrjc6zi"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-uvrjvengfn6b4if"},{"phid":"PHID-XACT-TASK-vaqifgtdx226wnz"},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"phid":"PHID-XACT-TASK-z4hdxqf7ponelsg"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-mwb5ciq2773xalj"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-efh4rww3pfynspw"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-s67l67gutsve6hxuigsu"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ipmp5o6h6oe4rrsato3h"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ctafnlifn64axvokqjy3"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-aylmuiyrxike6je"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-gjy3mfgsdcmzd76"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-wbvvnwoiufrdmy6"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-cwulfonylxg2gnv"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-xvcenvs5dt2kg7m"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ylarhsvhxcxqbur6pbmv"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-mvlwyd72obcvl6juxish"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-cm6l3muxcwmitvpzcc4u"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-6b6w5qogf4deywgbkuzs"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-ddeq3ikuxatonvi"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz"}},{"phid":"PHID-XACT-TASK-wa5fycx7amecwey"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-jnvlydqcsilmq75"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-fjm73r5bfmlqrh4"},{"hovercardSpec":{"objectPHID":"PHID-USER-z3l4i7dl52qicxtephy5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-nmlrftecxu3jfun"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"phid":"PHID-XACT-TASK-u5zccha2ohxr27e"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-kuaozpxrfizigpx"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-34vr7566qwogogn"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-hebxsobseocu47w"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-wmwti2dveg3svky"},{"hovercardSpec":{"objectPHID":"PHID-USER-xwxce3kpvsv54t735g7q"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz"}},{"phid":"PHID-XACT-TASK-afbziywhlog3alo"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-bxwerjgmy5jns6p"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-wr5m4klkq2p7edl"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-2v7rgubexjp2gfd"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-ivorqzrkfh6bzmh"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-ijwiav7lqi3tqbk"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-3dnumwtk2phscnl"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-rtwnn4xvdfy5p3y"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-n4o3mxspxi5z7or"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-bedfnnwm33glx5d"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-zfbpwqsf2gml6fz"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-q5vnn54dcmo6ess"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-vngzib3dbcm3grh"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-txxgas5wiyrmpq6"},{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"phid":"PHID-XACT-TASK-3lktxca2w743sbw"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-w3c7h2b6hm3sxxg"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-xj5w5wk22j63mwq"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-zflthyyra2oep4t"},{"hovercardSpec":{"objectPHID":"PHID-USER-tliyqayknga4r5aqqod6"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-fkraldlzsu5aeoh"},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"phid":"PHID-XACT-TASK-egkermsuy2vwhhe"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-erjsbfdlhsgbwq4","anchor":"6511463"},{"tip":"Via Herald"},[],[],{"phid":"PHID-XACT-TASK-wlxa7vvpjjdkxfs","anchor":"6511473"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-iidqqm4umcxxuld\/","ref":"T264398#6511496"},[],{"anchor":"6511496"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_206\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_207\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-iidqqm4umcxxuld\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_208\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-iidqqm4umcxxuld","anchor":"6511496"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-l4q7np7bdbbq4kz\/","ref":"T264398#6511513"},[],{"anchor":"6511513"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_215\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l4q7np7bdbbq4kz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_217\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-l4q7np7bdbbq4kz","anchor":"6511513"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mb2mtffrbeuh5rl","anchor":"6511515"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7r4g7tfwsr4qsf2","anchor":"6511533"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gvtmio6mk2px2w5\/","ref":"T264398#6512035"},[],{"anchor":"6512035"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_230\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gvtmio6mk2px2w5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_232\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gvtmio6mk2px2w5","anchor":"6512035"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uwfs3sfbbrjc6zi\/","ref":"T264398#6512038"},[],{"anchor":"6512038"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_239\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uwfs3sfbbrjc6zi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_241\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-65lvg6xs2dsxkz3","anchor":"6512038"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uvrjvengfn6b4if\/","ref":"T264398#6512255"},[],{"anchor":"6512255"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_248\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uvrjvengfn6b4if\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_250\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uvrjvengfn6b4if","anchor":"6512255"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vaqifgtdx226wnz\/","ref":"T264398#6512490"},[],{"anchor":"6512490"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_257\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_258\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vaqifgtdx226wnz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_259\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vaqifgtdx226wnz","anchor":"6512490"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-z4hdxqf7ponelsg\/","ref":"T264398#6512535"},[],{"anchor":"6512535"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_266\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-z4hdxqf7ponelsg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_268\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z4hdxqf7ponelsg","anchor":"6512535"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mwb5ciq2773xalj\/","ref":"T264398#6516308"},[],{"anchor":"6516308"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_275\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mwb5ciq2773xalj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_277\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mwb5ciq2773xalj","anchor":"6516308"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-efh4rww3pfynspw\/","ref":"T264398#6516328"},[],{"anchor":"6516328"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_284\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-efh4rww3pfynspw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_286\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-efh4rww3pfynspw\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_288\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-efh4rww3pfynspw","anchor":"6516328"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qcvopjbodlrmd2a","anchor":"6516552"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-aylmuiyrxike6je\/","ref":"T264398#6516644"},[],{"anchor":"6516644"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_297\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aylmuiyrxike6je\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_299\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-aylmuiyrxike6je","anchor":"6516644"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-yxp74qgbvc7tcbl","anchor":"6516645"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gjy3mfgsdcmzd76\/","ref":"T264398#6516651"},[],{"anchor":"6516651"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_309\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gjy3mfgsdcmzd76\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_311\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gjy3mfgsdcmzd76","anchor":"6516651"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wbvvnwoiufrdmy6\/","ref":"T264398#6516672"},[],{"anchor":"6516672"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_318\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wbvvnwoiufrdmy6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_320\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-wbvvnwoiufrdmy6","anchor":"6516672"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cwulfonylxg2gnv\/","ref":"T264398#6516680"},[],{"anchor":"6516680"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cwulfonylxg2gnv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_329\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cwulfonylxg2gnv","anchor":"6516680"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xvcenvs5dt2kg7m\/","ref":"T264398#6516730"},[],{"anchor":"6516730"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_336\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xvcenvs5dt2kg7m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_338\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xvcenvs5dt2kg7m","anchor":"6516730"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-y6vtgpqddubmkhh","anchor":"6516782"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-gxs44g2s7yk3odu","anchor":"6518865"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-77ml7bclyoc3yx6","anchor":"6518936"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ddeq3ikuxatonvi\/","ref":"T264398#6521346"},[],{"anchor":"6521346"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_355\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_356\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ddeq3ikuxatonvi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_357\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_358\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ddeq3ikuxatonvi","anchor":"6521346"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wa5fycx7amecwey\/","ref":"T264398#6524392"},[],{"anchor":"6524392"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_364\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wa5fycx7amecwey\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_366\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_367\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-a5u2ewfeultpw7u","anchor":"6524392"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jnvlydqcsilmq75\/","ref":"T264398#6524473"},[],{"anchor":"6524473"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_373\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_374\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jnvlydqcsilmq75\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_375\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_376\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jnvlydqcsilmq75","anchor":"6524473"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fjm73r5bfmlqrh4\/","ref":"T264398#6524537"},[],{"anchor":"6524537"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_382\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_383\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fjm73r5bfmlqrh4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_384\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_385\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-fjm73r5bfmlqrh4\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fjm73r5bfmlqrh4","anchor":"6524537"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-f3ljozzhmcpfyry","anchor":"6525611"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-7kqplbbbk6dkbyc","anchor":"6525955"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nmlrftecxu3jfun\/","ref":"T264398#6525968"},[],{"anchor":"6525968"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_399\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_400\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nmlrftecxu3jfun\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_401\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_402\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nmlrftecxu3jfun","anchor":"6525968"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kqjqvfwbukqebut","anchor":"6526044"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u5zccha2ohxr27e\/","ref":"T264398#6526187"},[],{"anchor":"6526187"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_411\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_412\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u5zccha2ohxr27e\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_413\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_414\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u5zccha2ohxr27e","anchor":"6526187"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kuaozpxrfizigpx\/","ref":"T264398#6527005"},[],{"anchor":"6527005"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_420\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_421\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kuaozpxrfizigpx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_422\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_423\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kuaozpxrfizigpx","anchor":"6527005"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-34vr7566qwogogn\/","ref":"T264398#6527983"},[],{"anchor":"6527983"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_429\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_430\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-34vr7566qwogogn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_431\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_432\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-34vr7566qwogogn","anchor":"6527983"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hebxsobseocu47w\/","ref":"T264398#6528126"},[],{"anchor":"6528126"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_438\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_439\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hebxsobseocu47w\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_440\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_441\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hebxsobseocu47w","anchor":"6528126"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wmwti2dveg3svky\/","ref":"T264398#6528203"},[],{"anchor":"6528203"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_447\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_448\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wmwti2dveg3svky\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_449\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_450\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-wmwti2dveg3svky\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wmwti2dveg3svky","anchor":"6528203"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-oncm2fitg353dqo","anchor":"6528373"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-afbziywhlog3alo\/","ref":"T264398#6528886"},[],{"anchor":"6528886"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_460\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_461\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-afbziywhlog3alo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_462\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_463\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-afbziywhlog3alo","anchor":"6528886"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bxwerjgmy5jns6p\/","ref":"T264398#6531798"},[],{"anchor":"6531798"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_469\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_470\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bxwerjgmy5jns6p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_471\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_472\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bxwerjgmy5jns6p","anchor":"6531798"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wr5m4klkq2p7edl\/","ref":"T264398#6532701"},[],{"anchor":"6532701"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_478\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_479\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wr5m4klkq2p7edl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_480\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_481\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wr5m4klkq2p7edl","anchor":"6532701"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2v7rgubexjp2gfd\/","ref":"T264398#6535510"},[],{"anchor":"6535510"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_487\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_488\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2v7rgubexjp2gfd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_489\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_490\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-2v7rgubexjp2gfd\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_491\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2v7rgubexjp2gfd","anchor":"6535510"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ivorqzrkfh6bzmh\/","ref":"T264398#6535643"},[],{"anchor":"6535643"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_497\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_498\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ivorqzrkfh6bzmh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_499\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_500\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ivorqzrkfh6bzmh","anchor":"6535643"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ijwiav7lqi3tqbk\/","ref":"T264398#6535694"},[],{"anchor":"6535694"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_506\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_507\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ijwiav7lqi3tqbk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_508\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_509\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ijwiav7lqi3tqbk","anchor":"6535694"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3dnumwtk2phscnl\/","ref":"T264398#6535852"},[],{"anchor":"6535852"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_515\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_516\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3dnumwtk2phscnl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_517\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_518\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3dnumwtk2phscnl\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_519\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3dnumwtk2phscnl","anchor":"6535852"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rtwnn4xvdfy5p3y\/","ref":"T264398#6535882"},[],{"anchor":"6535882"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_525\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_526\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rtwnn4xvdfy5p3y\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_527\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_528\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-rtwnn4xvdfy5p3y\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_529\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rtwnn4xvdfy5p3y","anchor":"6535882"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-n4o3mxspxi5z7or\/","ref":"T264398#6536020"},[],{"anchor":"6536020"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_535\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-n4o3mxspxi5z7or\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_537\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_538\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-n4o3mxspxi5z7or","anchor":"6536020"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bedfnnwm33glx5d\/","ref":"T264398#6536163"},[],{"anchor":"6536163"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_544\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_545\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bedfnnwm33glx5d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_546\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_547\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bedfnnwm33glx5d","anchor":"6536163"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zfbpwqsf2gml6fz\/","ref":"T264398#6536169"},[],{"anchor":"6536169"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_553\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_554\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zfbpwqsf2gml6fz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_555\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_556\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-zfbpwqsf2gml6fz\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_557\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-zfbpwqsf2gml6fz","anchor":"6536169"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-q5vnn54dcmo6ess\/","ref":"T264398#6536565"},[],{"anchor":"6536565"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_563\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-q5vnn54dcmo6ess\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_565\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-q5vnn54dcmo6ess","anchor":"6536565"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vngzib3dbcm3grh\/","ref":"T264398#6538699"},[],{"anchor":"6538699"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_572\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vngzib3dbcm3grh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_574\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_575\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-vngzib3dbcm3grh\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_576\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vngzib3dbcm3grh","anchor":"6538699"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-txxgas5wiyrmpq6\/","ref":"T264398#6563442"},[],{"anchor":"6563442"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_582\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_583\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-txxgas5wiyrmpq6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_584\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_585\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-txxgas5wiyrmpq6","anchor":"6563442"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3lktxca2w743sbw\/","ref":"T264398#6564065"},[],{"anchor":"6564065"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_591\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_592\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3lktxca2w743sbw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_593\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_594\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3lktxca2w743sbw","anchor":"6564065"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-w3c7h2b6hm3sxxg\/","ref":"T264398#6564285"},[],{"anchor":"6564285"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_600\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_601\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-w3c7h2b6hm3sxxg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_602\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_603\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-w3c7h2b6hm3sxxg","anchor":"6564285"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dfoxqtqh2lea4pc","anchor":"6564286"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xj5w5wk22j63mwq\/","ref":"T264398#6564303"},[],{"anchor":"6564303"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_612\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_613\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xj5w5wk22j63mwq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_614\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_615\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xj5w5wk22j63mwq","anchor":"6564303"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-zflthyyra2oep4t\/","ref":"T264398#6564517"},[],{"anchor":"6564517"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_621\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_622\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-zflthyyra2oep4t\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_623\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_624\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-zflthyyra2oep4t","anchor":"6564517"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dvvzutete5cozv7","anchor":"6564701"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fkraldlzsu5aeoh\/","ref":"T264398#6564734"},[],{"anchor":"6564734"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_633\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_634\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fkraldlzsu5aeoh\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_635\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_636\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-fkraldlzsu5aeoh\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_637\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fkraldlzsu5aeoh","anchor":"6564734"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-egkermsuy2vwhhe\/","ref":"T264398#6565046"},[],{"anchor":"6565046"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_643\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_644\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-egkermsuy2vwhhe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_645\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_646\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-egkermsuy2vwhhe","anchor":"6565046"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cc8a2c6b\/core.pkg.css"]}