for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-q7rcpaxadgsevowgok22\/?after=4816882&quoteTargetID=UQ0_1&quoteRef=T211661\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_211\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/7z4x3tw3vztolgui6ccw\/PHID-FILE-pqsnnzaawmxtemcuh4ik\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ema\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4816882\" id=\"4816882\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_210\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ema\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e ema\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1201\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_1\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1201\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003eRadar\/Not for service by Traffic\u003c\/a\u003e on the \u003ca href=\"\/tag\/traffic\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003eTraffic\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4816882\" data-sigil=\"has-tooltip\" data-meta=\"0_209\"\u003e\u003cspan class=\"screen-only\"\u003eDec 12 2018, 2:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-12 14:22:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_214\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rus3onc75davarls6nbc\/PHID-FILE-d4tltleyv665iw455lvp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Imarlier\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4829175\" id=\"4829175\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_213\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Imarlier\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Imarlier\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eInbox, needs triage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eTo-do: Goals, prioritized next 4 Quarters\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4829175\" data-sigil=\"has-tooltip\" data-meta=\"0_212\"\u003e\u003cspan class=\"screen-only\"\u003eDec 17 2018, 9:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-17 21:19:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_217\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4836858\" id=\"4836858\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_216\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e mentioned this in \u003ca href=\"\/T27611\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003eT27611: Serve optimized WebP thumbnails as alternative to JPEG, PNG\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4836858\" data-sigil=\"has-tooltip\" data-meta=\"0_215\"\u003e\u003cspan class=\"screen-only\"\u003eDec 20 2018, 9:50 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-12-20 09:50:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_220\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2rlpinqyz4xsjmfbcpuo\/PHID-FILE-c2cblouyyjoo7fapx3ek\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Phabricator_maintenance\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4912791\" id=\"4912791\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_219\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Phabricator_maintenance\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Phabricator_maintenance\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1025\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003eBacklog\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1025\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eAcknowledged\u003c\/a\u003e on the \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eSRE\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4912791\" data-sigil=\"has-tooltip\" data-meta=\"0_218\"\u003e\u003cspan class=\"screen-only\"\u003eJan 26 2019, 10:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-01-26 22:51:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_229\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4926909\" id=\"4926909\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_228\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4926909\" data-sigil=\"has-tooltip\" data-meta=\"0_227\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 11:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 11:19:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_225\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_226\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've looked at bloomd's capabilities and it doesn't have a concept of things expiring, which makes sense given the nature of a bloom filter.\u003c\/p\u003e\n\n\u003cp\u003eWhat if we had 30 instances of bloomd running at once, each started one day after the other? I.e. first bloomd instance was started 30 days ago, second one 29 days ago, etc. When thumbnail hits happen, we send updates to all 30 of them, but when we want to query the filter, we always look up the oldest bloomd instance. Every day we shut down and clean up the oldest instance and start a new one.\u003c\/p\u003e\n\n\u003cp\u003eThey can all persist on disk, which will make them resistant to reboots. This is very important to retain the 30 days of history.\u003c\/p\u003e\n\n\u003cp\u003eThe big unknown is how memory-intensive a given instance would be in practice, given the very long tail of content we have, which would have to be multiplied by approximately 30. I guess we can start with a single instance of bloomd on a codfw machine and seeing how large it grows over time?\u003c\/p\u003e\n\n\u003cp\u003eTo mitigate the duplicated memory use, we can also reduce the granularity, to say, 5 weekly instances. One started at most 35 days ago, the second one 28 days ago, etc. Which means that when the 35-day-old instance gets nuked, we know about the last 28 days of thumbnails. And we keep at most 35 days worth of memory (in the logical sense) about thumbnail hits.\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e what do you think of this approach of rolling bloomd instances?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_232\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/gjkyraeh6ascv7vkj3ke\/PHID-FILE-czquixv3lk45jved74s2\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/CDanis\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4927888\" id=\"4927888\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/CDanis\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003eCDanis\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4927888\" data-sigil=\"has-tooltip\" data-meta=\"0_230\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 4:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 16:29:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_243\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_241\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928551\" id=\"4928551\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003eori\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928551\" data-sigil=\"has-tooltip\" data-meta=\"0_239\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 6:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 18:19:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_237\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_238\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWhat orders of magnitude are we talking about with respect to: total number of thumbnails in Swift, and number of thumbnail requests per second?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_254\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928637\" id=\"4928637\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003eori\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928637\" data-sigil=\"has-tooltip\" data-meta=\"0_250\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 6:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 18:38:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_248\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_54\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt seems that Swift has \u003ca href=\"https:\/\/docs.openstack.org\/swift\/latest\/api\/object-expiration.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebuilt-in support for object expiration\u003c\/a\u003e, which can be requested by setting a header (either \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After\u003c\/tt\u003e or \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-At\u003c\/tt\u003e).\u003cbr \/\u003e\nIt also looks like the expiry can be re-set, either by first removing it via \u003ctt class=\"remarkup-monospaced\"\u003eX-Remove-Delete-At\u003c\/tt\u003e, and then setting it anew, or by \u003ca href=\"https:\/\/stackoverflow.com\/a\/43723273\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eupdating the metadata in-place\u003c\/a\u003e.\u003cbr \/\u003e\nIs there a reason why these mechanisms are not under consideration?\u003c\/p\u003e\n\n\u003cp\u003eA straw-man proposal:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOn thumbnail creation, set \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After: 2592000\u003c\/tt\u003e (one month).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEach time a thumbnail is retrieved, there's a 0.01% chance we also used the opportunity to reset the expiry to one month.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_263\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928797\" id=\"4928797\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928797\" data-sigil=\"has-tooltip\" data-meta=\"0_261\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 7:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 19:05:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_259\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_56\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThis hadn't been considered because I didn't know that feature existed... Checking on the production Swift servers, swift-object-expirer isn't available, which is probably why I didn't know about it. This looks perfect for our needs. I need to figure out which swift version came up with this feature and\/or whether it's an optional install of some kind on Debian.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_272\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928804\" id=\"4928804\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_271\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928804\" data-sigil=\"has-tooltip\" data-meta=\"0_270\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 7:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 19:07:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_268\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_58\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWell, seems to have been that simple all along: \u003ca href=\"https:\/\/packages.debian.org\/jessie-backports\/swift-object-expirer\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/packages.debian.org\/jessie-backports\/swift-object-expirer\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_281\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928838\" id=\"4928838\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_280\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928838\" data-sigil=\"has-tooltip\" data-meta=\"0_279\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 7:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 19:12:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_277\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_60\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFor setting it on X% of access, preferably async, we'll just need a little logic to check that the thumbnail has a corresponding original. On thumbnail creation that's not needed, I'll probably start by implementing that part.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_292\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_290\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_291\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928875\" id=\"4928875\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003eori\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928875\" data-sigil=\"has-tooltip\" data-meta=\"0_288\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 7:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 19:22:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_286\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e(It might have to be X% of access \u003cem\u003eon varnish\u003c\/em\u003e, since I assume the most oft-requested thumbnails enjoy a very high varnish cache hit rate. You could do this asynchronously by having a daemon that samples thumbnail requests from the varnishkafka stream.)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_301\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4928884\" id=\"4928884\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_300\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/BBlack\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003eBBlack\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4928884\" data-sigil=\"has-tooltip\" data-meta=\"0_299\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 7:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 19:25:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_297\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe discussed this a bunch and we believe that basing this on Swift access alone with an expiry of a month is fine. The hottest things can't stay in Varnish for a month, even if you combine the longest lifetime possible at every Varnish layer. I can't remember what the combined Varnish expiry values were, but this is what \u003ca href=\"\/p\/BBlack\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@BBlack\u003c\/span\u003e\u003c\/a\u003e said.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_312\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_310\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_311\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4929122\" id=\"4929122\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_309\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003eori\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4929122\" data-sigil=\"has-tooltip\" data-meta=\"0_308\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 5 2019, 7:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-05 19:51:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_306\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe worry I had was this: a thumbnail that is requested once a minute on average probably has an approximately similar varnish cache hit rate to a thumbnail that is requested a hundred times per second. If that's true, then both thumbnails would be retrieved from Swift about as often, and they would be equally likely to have their expiry renewed before they're vacuumed up. This increases the risk of overload in case of a varnish cold restart or failure.\u003c\/p\u003e\n\n\u003cp\u003eIf you cranked the Swift retrieval sampling factor to a high enough value, this wouldn't matter, because both the once-per-minute and 100-per-second thumbnails would be virtually guaranteed to get refreshed before the 30 days are up. But then you're doing a lot more metadata updates in Swift than you really need to. You could probably get away with doing much fewer metadata updates if you sampled varnish requests instead.\u003c\/p\u003e\n\n\u003cp\u003eThere's also a fair bit of experience at the WMF in setting up and operating varnishlog-tailing daemons, of which this would be an instance. Is there a similarly robust way of tailing the swift req logs? (My memory is very rusty...)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_321\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4930751\" id=\"4930751\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4930751\" data-sigil=\"has-tooltip\" data-meta=\"0_319\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 8:51 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 08:51:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_317\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_318\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe simplest architecture really is to touch swift objects on every retrieval, which can be done async, but the unknown is how much extra load having every read causing a write would cause. I think we can easily find out by applying this logic to only one Commons thumbnail shard, or containers of smaller wikis. Same thing for experimenting with a percentage of requests triggering touches and seeing how much extra thumbor load that might cause.\u003c\/p\u003e\n\n\u003cp\u003eThe issue of tailing logs, be it varnish or swift, is that we have a very long tail of URLs being hit, and you're pretty much back to bloom filters being the optimal solution, in which case we end up doing what I suggested earlier.\u003c\/p\u003e\n\n\u003cp\u003eI think it's worth trying the "simple thing" first on a portion of the traffic, as we might worry too much about the effects of updating a header asynchronously on each read. Even though it is a scary-looking 100x increase in write operations.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_330\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4931056\" id=\"4931056\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_329\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4931056\" data-sigil=\"has-tooltip\" data-meta=\"0_328\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 10:41 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 10:41:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_326\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_327\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIIRC object expiration was considered years ago (i.e. \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Swift\/ObjectExpiration\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Swift\/ObjectExpiration\u003c\/a\u003e) and at the time considered buggy for production use, though after 4+ years I'm sure the situation has changed so definitely worth reconsidering as an option too.\u003c\/p\u003e\n\n\u003cp\u003eTo answer a few questions:\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4929122\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4929122\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThere's also a fair bit of experience at the WMF in setting up and operating varnishlog-tailing daemons, of which this would be an instance. Is there a similarly robust way of tailing the swift req logs? (My memory is very rusty...)\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eSwift access logs ATM are stored only on swift proxies themselves, though should be reasonably easy to tail\/process them nowadays.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4928538\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4928538\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eWhat orders of magnitude are we talking about with respect to: total number of thumbnails in Swift, and number of thumbnail requests per second?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eATM there's ~1.2B objects in thumbnails containers, re: thumbnails requested per second I don't have a breakdown handy though swift sees a peak of 1.5k requests a second (2xx+3xx that is) which I'm assuming thumbnails are the majority. For writes it is about 100x less than that (i.e. ~10\/s) \u003ca href=\"https:\/\/grafana.wikimedia.org\/d\/OPgmB1Eiz\/swift?orgId=1&from=now-7d&to=now-1m\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana.wikimedia.org\/d\/OPgmB1Eiz\/swift?orgId=1&from=now-7d&to=now-1m\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eAnd indeed I share the concerns already mentioned, namely making sure we're able to have a bound on writes (deletes and\/or metadata updates) as we're expiring thumbnails.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_341\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_339\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_340\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4931840\" id=\"4931840\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_338\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003eori\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4931840\" data-sigil=\"has-tooltip\" data-meta=\"0_337\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 3:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 15:35:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_335\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_336\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4931056\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4931056\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAnd indeed I share the concerns already mentioned, namely making sure we're able to have a bound on writes (deletes and\/or metadata updates) as we're expiring thumbnails.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eHi Filippo! :) Thanks for humoring my kibbitzing.\u003c\/p\u003e\n\n\u003cp\u003eYou could update metadata on 0.01% of requests, but cap that to a fixed maximum number of metadata updates per minute. And if your expiry times are relative (using \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After\u003c\/tt\u003e), the limit on metadata updates also gives you an upper bound on the rate of deletes. (But it also means that any time you tweak the rate of metadata updates, there is a delay of \u003ctt class=\"remarkup-monospaced\"\u003e$TTL\u003c\/tt\u003e seconds before you see the effect on deletions.) I figure you'd start with a very conservative cap and then bump it up over time.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_344\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4931971\" id=\"4931971\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_343\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003ejijiki\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4931971\" data-sigil=\"has-tooltip\" data-meta=\"0_342\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 4:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 16:13:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_353\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4932147\" id=\"4932147\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_352\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4932147\" data-sigil=\"has-tooltip\" data-meta=\"0_351\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 4:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 16:46:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_349\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_76\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt's true that even if we only clean up a portion of thumbnails, we're already in a good place. The operational goal is to free up space at a rate fast enough to allow introducing new variants (eg. WebP). Ideally we can crank this up high enough that total storage use goes down over time, even slightly.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_363\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4932156\" id=\"4932156\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#4932156\" data-sigil=\"has-tooltip\" data-meta=\"0_361\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 4:49 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 16:49:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_359\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_360\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_78\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWhat we can do is if we see that the thumbnail already has a X-Delete-After header on get, we update the header. If it doesn't have the header yet, we roll the dice. This avoids the potential issue of a touched thumbnail appearing to be unused just because the subsequent times it was accessed, it didn't get (randomly) touched.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_366\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/dehmh3scjpmoumxznua7\/PHID-FILE-4esn6uidadgmq7ck5yod\/alphanumeric_aleo-white_M.png-_b38ba9-0%2C0%2C0%2C0.3.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/MoritzMuehlenhoff\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4932257\" id=\"4932257\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_365\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/MoritzMuehlenhoff\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eMoritzMuehlenhoff\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4932257\" data-sigil=\"has-tooltip\" data-meta=\"0_364\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 5:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 17:27:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_369\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/hg7ps6fmrrem7raik7y7\/PHID-FILE-n24znlpfcck36hydc7ff\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jbond\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4932258\" id=\"4932258\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jbond\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003ejbond\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4932258\" data-sigil=\"has-tooltip\" data-meta=\"0_367\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 6 2019, 5:28 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-06 17:28:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_378\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4934095\" id=\"4934095\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4934095\" data-sigil=\"has-tooltip\" data-meta=\"0_376\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 9:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 09:37:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_374\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_375\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4931840\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4931840\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4931056\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4931056\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAnd indeed I share the concerns already mentioned, namely making sure we're able to have a bound on writes (deletes and\/or metadata updates) as we're expiring thumbnails.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eHi Filippo! :) Thanks for humoring my kibbitzing.\u003c\/p\u003e\n\n\u003cp\u003eYou could update metadata on 0.01% of requests, but cap that to a fixed maximum number of metadata updates per minute. And if your expiry times are relative (using \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After\u003c\/tt\u003e), the limit on metadata updates also gives you an upper bound on the rate of deletes. (But it also means that any time you tweak the rate of metadata updates, there is a delay of \u003ctt class=\"remarkup-monospaced\"\u003e$TTL\u003c\/tt\u003e seconds before you see the effect on deletions.) I figure you'd start with a very conservative cap and then bump it up over time.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eHi Ori :) No problem -- thanks for chiming in, appreciated!\u003c\/p\u003e\n\n\u003cp\u003eAgreed starting with a very small percentage of requests (as seen by swift, aiui) and set deletion is a good knob to have.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4932156\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4932156\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eWhat we can do is if we see that the thumbnail already has a X-Delete-After header on get, we update the header. If it doesn't have the header yet, we roll the dice. This avoids the potential issue of a touched thumbnail appearing to be unused just because the subsequent times it was accessed, it didn't get (randomly) touched.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eSounds like a good optimization going forward, though how we would limit the header updates when a large number of thumbnails being fetched has set x-delete-after ?\u003c\/p\u003e\n\n\u003cp\u003eAlso AIUI we'd do this work in swift proxy not varnish?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_387\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4934161\" id=\"4934161\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4934161\" data-sigil=\"has-tooltip\" data-meta=\"0_385\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 10:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 10:14:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_383\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_384\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI thought we would start with a very low percentage and ramp it up gradually. And yes, I thought our beloved swift proxy is where it would live.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_396\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4934164\" id=\"4934164\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_395\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003ejijiki\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4934164\" data-sigil=\"has-tooltip\" data-meta=\"0_394\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 10:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 10:16:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_392\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_393\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIs it possible to hold this a bit for until after we upgrade all Thumbor servers to stretch? Two birds with one stone :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_405\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4934183\" id=\"4934183\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_87\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4934183\" data-sigil=\"has-tooltip\" data-meta=\"0_403\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 10:23 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 10:23:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_401\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_402\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_88\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI'd argue that we don't want both changes to happen around the same time. And this is probably less prone to emergency bugfixes than the Stretch upgrade would be. I would rather see this deployed first, as the Stretch upgrade will probably create a trickle of bugs to fix after it's deployed (potential SVG\/PDF rendering regressions, etc.).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_414\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4934261\" id=\"4934261\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_413\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_89\"\u003ejijiki\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4934261\" data-sigil=\"has-tooltip\" data-meta=\"0_412\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 11:02 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 11:02:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_410\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_411\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_90\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4934183\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4934183\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI'd argue that we don't want both changes to happen around the same time. And this is probably less prone to emergency bugfixes than the Stretch upgrade would be. I would rather see this deployed first, as the Stretch upgrade will probably create a trickle of bugs to fix after it's deployed (potential SVG\/PDF rendering regressions, etc.).\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYou have a point, +1.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_426\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_424\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_425\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4935167\" id=\"4935167\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_423\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003eori\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#4935167\" data-sigil=\"has-tooltip\" data-meta=\"0_422\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 4:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 16:07:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_420\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_421\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI don't understand the preference for sampling Swift requests rather than Varnish requests. You'd have greater resilience to overload (for the reasons I \u003ca href=\"#anchor-4931840\" class=\"remarkup-link\" rel=\"noreferrer\"\u003ecited above\u003c\/a\u003e), and you'd have looser coupling and better fault tolerance by building on top of kafka.\u003c\/p\u003e\n\n\u003cp\u003eI'm also not sure I understand the point of checking whether the image already has an expiry header or not. Is the idea that if an image doesn't have the header, then we don't have to worry about it expiring? That's true, but:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eInitially\u003c\/em\u003e, \u003cstrong\u003eno\u003c\/strong\u003e thumbnail will have the expiry header, so without some additional criterion for when you should set the header, you'll never expire anything.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003cem\u003eEventually\u003c\/em\u003e, \u003cstrong\u003eevery\u003c\/strong\u003e thumbnail will have the expiry header, so once the initial ramp-up is done, this is no longer a relevant signal at all.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSo you're getting very little value out of this approach IMO, and it forces you to read \u003cem\u003eresponse\u003c\/em\u003e headers from Swift. By looking at responses coming from Swift rather than requests coming in to Varnish, you don't have a good signal for what the hottest thumbnails are, since somewhat-popular thumbnails will be retrieved from Swift about as often as extra-super-popular thumbnails.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_436\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4936243\" id=\"4936243\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_435\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_93\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#4936243\" data-sigil=\"has-tooltip\" data-meta=\"0_434\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 8:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 20:37:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_432\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_433\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_94\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe "initial ramp up" might not ever be done, if we reach a point where the writes and deletes introduced are creating too much overhead, we'll settle for only cleaning up some of the thumbnails, which is absolutely fine. If we run into that situation, we may stay in it for a while, but it's already an improvement compared to the status quo. Maybe we will increase the Swift capacity to allow us to clean up everything, maybe we won't. The goal is freeing up space\/reducing the storage growth to introduce new features.\u003c\/p\u003e\n\n\u003cp\u003eWithout knowing the effects any cleanup strategy will have ahead of time, only doing a partial ongoing cleanup is the most likely scenario, until it's demonstrated that it's fine to do fully for 100% of thumbnails.\u003c\/p\u003e\n\n\u003cp\u003eIf we try this out on a portion of the traffic with headers, once we put a thumbnail in the experiment by setting a header, it needs to stay in it. Otherwise it might look expired when it isn't, if you only touch it randomly.\u003c\/p\u003e\n\n\u003cp\u003eBeing blind to Varnish is absolutely inconsequential when the hottest thumbnails will get renewed several times in the 30-day window, because they're guaranteed to fall out of Varnish multiple times, regardless of how hot they are. Any individual image can't stick around in the Varnish layers for more than a few days, even if hit constantly. By the time the object expirer gets to an object that hasn't been touched in 30 days, it's guaranteed not to be a hot thumbnail from Varnish's perspective. So you're getting that guarantee indirectly, without any need to record or parse any traffic stats, just by virtue of the relationship between the different expiry durations for the Varnish and Swift layers.\u003c\/p\u003e\n\n\u003cp\u003eWhat matters is not cleaning up a thumbnail that's hot at the edge, and you get that by waiting long enough that it would have been fetched from Swift multiple times if that was the case. Once you reach 30 days of being untouched, you know for a fact that nobody has accessed it during that period minus the few days something can stay in Varnish.\u003c\/p\u003e\n\n\u003cp\u003eI already have the whole Swift-based setup working in Vagrant as of earlier today, it'll be up for review soon, once I've cleaned up the hardcoded stuff. It's very simple, which is why I think it's the way to go for an initial attempt. Since we have no idea what the impact will be, it's also possible that Swift won't break a sweat with the extra header writes and deletes. I don't see the point of building a more complicated solution based on "what ifs". If this approach doesn't work out, then it makes sense to consider a more complex setup with more services involved.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_445\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4936372\" id=\"4936372\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_444\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003egerritbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4936372\" data-sigil=\"has-tooltip\" data-meta=\"0_443\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 9:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 21:16:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_441\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_442\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 489021 had a related patch set uploaded (by Gilles; owner: Gilles):\u003cbr \/\u003e\n[mediawiki\/vagrant@master] Make Swift renew X-Delete headers when encountered\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/489021\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/489021\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_454\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4936373\" id=\"4936373\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_453\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_97\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4936373\" data-sigil=\"has-tooltip\" data-meta=\"0_452\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 9:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 21:16:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_450\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_451\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 489022 had a related patch set uploaded (by Gilles; owner: Gilles):\u003cbr \/\u003e\n[operations\/software\/thumbor-plugins@master] Set expiry headers on thumbnails\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/489022\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/489022\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_457\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4936427\" id=\"4936427\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_456\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e mentioned this in \u003ca href=\"\/rTHMBREXTb4c07418c289f244f217866e8eecdfd0c2066efd\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_101\"\u003erTHMBREXTb4c07418c289: Set expiry headers on thumbnails\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4936427\" data-sigil=\"has-tooltip\" data-meta=\"0_455\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 7 2019, 9:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-07 21:29:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_460\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4944275\" id=\"4944275\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_459\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e mentioned this in \u003ca href=\"\/rTHMBREXT6e6aacf5b25af1bce7db839f3b9a6922f0ee5f02\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_103\"\u003erTHMBREXT6e6aacf5b25a: Set expiry headers on thumbnails\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4944275\" data-sigil=\"has-tooltip\" data-meta=\"0_458\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 11 2019, 4:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-11 16:54:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_463\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/st3g3yziltbwjorfzd75\/PHID-FILE-d5zes5mjcqa2dfoswe23\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Peter\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4945314\" id=\"4945314\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_462\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Peter\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003ePeter\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4945314\" data-sigil=\"has-tooltip\" data-meta=\"0_461\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 11 2019, 9:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-11 21:15:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_466\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4950335\" id=\"4950335\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_465\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_105\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e mentioned this in \u003ca href=\"\/rTHMBREXT10d8dd8645ae6fbe709d102f17a9b6244e357d5a\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003erTHMBREXT10d8dd8645ae: Set expiry headers on thumbnails\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4950335\" data-sigil=\"has-tooltip\" data-meta=\"0_464\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 13 2019, 9:29 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-13 09:29:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_475\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/kd4xv6kmiqlmbg5i3ynf\/PHID-FILE-q756a47n7hkxmwqviotl\/profile-profile.jpg)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aaron\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4959116\" id=\"4959116\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_474\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eaaron\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4959116\" data-sigil=\"has-tooltip\" data-meta=\"0_473\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 16 2019, 8:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-16 08:49:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_471\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_472\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4928637\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4928637\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eIt seems that Swift has \u003ca href=\"https:\/\/docs.openstack.org\/swift\/latest\/api\/object-expiration.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ebuilt-in support for object expiration\u003c\/a\u003e, which can be requested by setting a header (either \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After\u003c\/tt\u003e or \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-At\u003c\/tt\u003e).\u003cbr \/\u003e\nIt also looks like the expiry can be re-set, either by first removing it via \u003ctt class=\"remarkup-monospaced\"\u003eX-Remove-Delete-At\u003c\/tt\u003e, and then setting it anew, or by \u003ca href=\"https:\/\/stackoverflow.com\/a\/43723273\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eupdating the metadata in-place\u003c\/a\u003e.\u003cbr \/\u003e\nIs there a reason why these mechanisms are not under consideration?\u003c\/p\u003e\n\n\u003cp\u003eA straw-man proposal:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eOn thumbnail creation, set \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After: 2592000\u003c\/tt\u003e (one month).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eEach time a thumbnail is retrieved, there's a 0.01% chance we also used the opportunity to reset the expiry to one month.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#4930751\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#4930751\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eThe simplest architecture really is to touch swift objects on every retrieval, which can be done async, but the unknown is how much extra load having every read causing a write would cause.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eHeh, this sounded like d\u00e9ja vu. Then I realized I had an old etherpad (\u003ca href=\"https:\/\/etherpad.wikimedia.org\/p\/swift-thumbnails\/timeslider#309\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/etherpad.wikimedia.org\/p\/swift-thumbnails\/timeslider#309\u003c\/a\u003e) linked from \u003ca href=\"\/T77697\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT77697\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e. Looking at that now, I notice that the "K hours" bits in that etherpad should really read "K days" since \u003cem\u003esome \u003c\/em\u003enon-trivial CDN\/Swift TTL delta is just the bare minimum to be able to guess "hotness" in common cases...there is of course no reason for the delta to merely be hours given the backup\/cold-cache use case. Also, missing from that etherpad is handling for requests coming in *sooner* than the CDN expiry (e.g. what if CDN was randomly cleared or several CDN nodes failed-over?).  I guess in that case, the POST could be skipped and the same X-Object-Meta-CDN-Expires-At use for the Expires header. I suppose Varnish "collapsed forwarding"\/"busy hashes" would avoid Swift POST stampedes for single URLs after CDN expiry. The added POST writes due to the size of the long-tail of cache misses for thumbnails that *are* in Swift might be tolerable.\u003c\/p\u003e\n\n\u003cp\u003eAnyway, for any non-percentage based scheme to POST new Swift TTLs (X-Delete-At), I'm wondering about malicious cases of someone spamming requests with cache busting URL parameters to millions of different thumbnails that \u003cem\u003ealready exists\u003c\/em\u003e in Swift (thus they would not trigger thumbor\/thumb.php throttling). Crawling some "most used files" list and hitting the standard sizes for thumbnails might be used for such malice. That could still cause unthrottled POST spam to Swift unless some rate-limiting is added.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_478\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/lmqso2pdiuhzvrx7fltm\/PHID-FILE-g4ayg62xazjsnsxkgiqt\/alphanumeric_lato-dark_L.png-_48a3ba-0%2C0%2C0%2C0.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Legoktm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4959157\" id=\"4959157\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_477\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Legoktm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_109\"\u003eLegoktm\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4959157\" data-sigil=\"has-tooltip\" data-meta=\"0_476\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 16 2019, 9:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-16 09:11:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_488\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4959380\" id=\"4959380\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_487\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#4959380\" data-sigil=\"has-tooltip\" data-meta=\"0_486\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 16 2019, 4:11 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-02-16 16:11:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_484\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_111\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eGood point. I think we just ought to clean up that loophole in Varnish, which is already removing the most obvious cache-busting attempts (GET parameters). If Varnish normalized the last part of the thumbnail URL, there wouldn't be a way to spam Swift for existing objects.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_491\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5015235\" id=\"5015235\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_490\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e mentioned this in \u003ca href=\"\/T180899\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_113\"\u003eT180899: PNG\/WebP thumbnails of SVG images have color issues in Firefox\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5015235\" data-sigil=\"has-tooltip\" data-meta=\"0_489\"\u003e\u003cspan class=\"screen-only\"\u003eMar 11 2019, 2:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-03-11 14:05:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_500\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5025597\" id=\"5025597\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_499\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5025597\" data-sigil=\"has-tooltip\" data-meta=\"0_498\"\u003e\u003cspan class=\"screen-only\"\u003eMar 14 2019, 8:13 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-03-14 20:13:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_496\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_497\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 489021 \u003cstrong\u003emerged\u003c\/strong\u003e by jenkins-bot:\u003cbr \/\u003e\n[mediawiki\/vagrant@master] Make Swift renew X-Delete headers when encountered\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/489021\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/489021\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-yellow\" data-sigil=\"transaction anchor-container\" data-meta=\"0_504\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5043052\" id=\"5043052\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-yellow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-spinner phui-timeline-icon\" data-meta=\"0_502\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e changed the task status from \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eStalled\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5043052\" data-sigil=\"has-tooltip\" data-meta=\"0_501\"\u003e\u003cspan class=\"screen-only\"\u003eMar 21 2019, 9:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-03-21 09:49:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_503\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_117\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e changed the status of subtask \u003ca href=\"\/T216339\" class=\"phui-handle handle-status-closed\"\u003eT216339: Normalize thumbnail request URLs in Varnish to avoid cachebusting\u003c\/a\u003e from \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eStalled\u003c\/span\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_507\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5089823\" id=\"5089823\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_506\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003ejijiki\u003c\/a\u003e mentioned this in \u003ca href=\"\/rTHMBREXT1ccde19798251cde195d9f2d9f541baa4f8892cc\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_119\"\u003erTHMBREXT1ccde1979825: Set expiry headers on thumbnails\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5089823\" data-sigil=\"has-tooltip\" data-meta=\"0_505\"\u003e\u003cspan class=\"screen-only\"\u003eApr 5 2019, 11:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-04-05 23:23:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_510\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5132317\" id=\"5132317\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_509\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_120\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_121\"\u003eTo-do: Goals, prioritized next 4 Quarters\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003eTo-do: Goals prioritized current Quarter\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_123\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5132317\" data-sigil=\"has-tooltip\" data-meta=\"0_508\"\u003e\u003cspan class=\"screen-only\"\u003eApr 23 2019, 8:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-04-23 20:05:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_513\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5268647\" id=\"5268647\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_512\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e changed the status of subtask \u003ca href=\"\/T216339\" class=\"phui-handle handle-status-closed\"\u003eT216339: Normalize thumbnail request URLs in Varnish to avoid cachebusting\u003c\/a\u003e from \u003cspan class=\"phui-timeline-value\"\u003eStalled\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5268647\" data-sigil=\"has-tooltip\" data-meta=\"0_511\"\u003e\u003cspan class=\"screen-only\"\u003eJun 19 2019, 2:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-19 14:10:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_523\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273185\" id=\"5273185\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-exclamation-circle phui-timeline-icon\" data-meta=\"0_521\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_125\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e changed the task status from \u003cspan class=\"phui-timeline-value\"\u003eStalled\u003c\/span\u003e to \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273185\" data-sigil=\"has-tooltip\" data-meta=\"0_520\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 8:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 08:22:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_522\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e closed subtask \u003ca href=\"\/T216339\" class=\"phui-handle handle-status-closed\"\u003eT216339: Normalize thumbnail request URLs in Varnish to avoid cachebusting\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_518\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_519\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_127\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/aaron\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aaron\u003c\/span\u003e\u003c\/a\u003e your concern has been addressed now, the Varnish-level thumbnail URL normalization is live. We can now proceed with the header-based expiry plan.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_532\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273190\" id=\"5273190\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_531\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_128\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273190\" data-sigil=\"has-tooltip\" data-meta=\"0_530\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 8:25 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 08:25:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_528\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_529\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_129\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@jijiki\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e Have Swift proxies been restarted since \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/#\/c\/mediawiki\/vagrant\/+\/489021\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/#\/c\/mediawiki\/vagrant\/+\/489021\/\u003c\/a\u003e was merged?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_541\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273209\" id=\"5273209\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_540\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003ejijiki\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273209\" data-sigil=\"has-tooltip\" data-meta=\"0_539\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 8:27 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 08:27:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_537\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_538\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@Gilles\u003c\/span\u003e\u003c\/a\u003e can it wait till next week when we will all be back, unless it is urgent, in which case we will figure it out\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_550\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273217\" id=\"5273217\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_549\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273217\" data-sigil=\"has-tooltip\" data-meta=\"0_548\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 8:30 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 08:30:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_546\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_547\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt can wait. Basically I want to figure out where we're at in regards to that patch, what's actually deployed and running.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_553\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5273218\" id=\"5273218\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_552\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003ejijiki\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/user-jijiki\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_135\"\u003eUser-jijiki\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273218\" data-sigil=\"has-tooltip\" data-meta=\"0_551\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 8:30 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 08:30:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_562\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273224\" id=\"5273224\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_561\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273224\" data-sigil=\"has-tooltip\" data-meta=\"0_560\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 8:35 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 08:35:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_558\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_559\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNevermind, this was the Vagrant patch... I'm going to make the production one now\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_571\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5273370\" id=\"5273370\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_570\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273370\" data-sigil=\"has-tooltip\" data-meta=\"0_569\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 9:16 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 09:16:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_567\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_568\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 518226 had a related patch set uploaded (by Gilles; owner: Gilles):\u003cbr \/\u003e\n[operations\/puppet@production] Have the Swift rewrite proxy renew expiry headers\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/518226\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/518226\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_574\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5273427\" id=\"5273427\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e reopened subtask \u003ca href=\"\/T216339\" class=\"phui-handle handle-status-closed\"\u003eT216339: Normalize thumbnail request URLs in Varnish to avoid cachebusting\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eOpen\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273427\" data-sigil=\"has-tooltip\" data-meta=\"0_572\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 9:45 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 09:45:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_577\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5273949\" id=\"5273949\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_576\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_141\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e closed subtask \u003ca href=\"\/T216339\" class=\"phui-handle handle-status-closed\"\u003eT216339: Normalize thumbnail request URLs in Varnish to avoid cachebusting\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5273949\" data-sigil=\"has-tooltip\" data-meta=\"0_575\"\u003e\u003cspan class=\"screen-only\"\u003eJun 21 2019, 1:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-21 13:58:11 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_586\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5288525\" id=\"5288525\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_585\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_142\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5288525\" data-sigil=\"has-tooltip\" data-meta=\"0_584\"\u003e\u003cspan class=\"screen-only\"\u003eJun 27 2019, 10:17 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-27 10:17:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_582\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_583\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 518226 \u003cstrong\u003emerged\u003c\/strong\u003e by Effie Mouzeli:\u003cbr \/\u003e\n[operations\/puppet@production] Have the Swift rewrite proxy renew expiry headers\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/518226\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/518226\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_595\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5288592\" id=\"5288592\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_594\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_144\"\u003eStashbot\u003c\/a\u003e mentioned this in \u003ca href=\"\/T226373\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_145\"\u003eT226373: Swift object servers become briefly unresponsive on a regular basis\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5288592\" data-sigil=\"has-tooltip\" data-meta=\"0_593\"\u003e\u003cspan class=\"screen-only\"\u003eJun 27 2019, 10:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-27 10:48:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_591\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_592\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_146\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWuYjROBOwpQ-3PktPyo\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_4\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-operations)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2019-06-27T10:48:30Z] <jijiki> Rolling restart ms-fe* proxy services for \u003ca href=\"\/T226373\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT226373\u003c\/span\u003e\u003c\/a\u003e and \u003ca href=\"\/T211661\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_31\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_604\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5288667\" id=\"5288667\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_603\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_147\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5288667\" data-sigil=\"has-tooltip\" data-meta=\"0_602\"\u003e\u003cspan class=\"screen-only\"\u003eJun 27 2019, 11:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-06-27 11:19:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_600\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_601\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_148\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 519374 had a related patch set uploaded (by Gilles; owner: Gilles):\u003cbr \/\u003e\n[operations\/puppet@production] Only apply expiry logic to "thumb" zone\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/519374\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/519374\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_607\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5297943\" id=\"5297943\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_606\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_149\"\u003eKrinkle\u003c\/a\u003e added a parent task: \u003ca href=\"\/T27611\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_150\"\u003eT27611: Serve optimized WebP thumbnails as alternative to JPEG, PNG\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5297943\" data-sigil=\"has-tooltip\" data-meta=\"0_605\"\u003e\u003cspan class=\"screen-only\"\u003eJul 1 2019, 8:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-01 20:20:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_610\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5535831\" id=\"5535831\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_609\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_151\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_152\"\u003eTo-do: Goals prioritized current Quarter\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_153\"\u003eBlocked (old)\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5535831\" data-sigil=\"has-tooltip\" data-meta=\"0_608\"\u003e\u003cspan class=\"screen-only\"\u003eSep 30 2019, 8:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-09-30 20:38:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_619\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5681381\" id=\"5681381\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_618\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_155\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5681381\" data-sigil=\"has-tooltip\" data-meta=\"0_617\"\u003e\u003cspan class=\"screen-only\"\u003eNov 21 2019, 1:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-11-21 13:00:19 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_615\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_616\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 519374 \u003cstrong\u003emerged\u003c\/strong\u003e by Effie Mouzeli:\u003cbr \/\u003e\n[operations\/puppet@production] Only apply expiry logic to "thumb" zone\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/519374\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/519374\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_628\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5770405\" id=\"5770405\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_627\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_157\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5770405\" data-sigil=\"has-tooltip\" data-meta=\"0_626\"\u003e\u003cspan class=\"screen-only\"\u003eJan 2 2020, 9:09 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-01-02 09:09:09 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_624\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_625\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_158\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnalytics now publishes media access stats, might be useful to drive some\/all thumbnail cleanup: \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Analytics\/AQS\/Mediarequests\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Analytics\/AQS\/Mediarequests\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_637\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5953594\" id=\"5953594\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_636\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_159\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_160\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5953594\" data-sigil=\"has-tooltip\" data-meta=\"0_635\"\u003e\u003cspan class=\"screen-only\"\u003eMar 9 2020, 12:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-03-09 12:51:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_633\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_634\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_161\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/dpifke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@dpifke\u003c\/span\u003e\u003c\/a\u003e this is something I might be interested in handing off to you and that you might want to consider for Q4. Let me know if that sounds like something up your alley.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_646\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/qdwewbx3e7yxhxche6ch\/PHID-FILE-522xul437qimyz7aedqw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dpifke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5954991\" id=\"5954991\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_645\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_162\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5954991\" data-sigil=\"has-tooltip\" data-meta=\"0_644\"\u003e\u003cspan class=\"screen-only\"\u003eMar 10 2020, 12:06 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-03-10 00:06:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_642\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_643\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_163\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYes, this is totally something I can tackle.  It meshes well with the work I'm doing on converting ArcLamp to use Swift for its storage.  (That task is slightly simplified by now knowing that \u003ctt class=\"remarkup-monospaced\"\u003eX-Delete-After\u003c\/tt\u003e exists.)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_655\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5955660\" id=\"5955660\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_654\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_164\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5955660\" data-sigil=\"has-tooltip\" data-meta=\"0_653\"\u003e\u003cspan class=\"screen-only\"\u003eMar 10 2020, 11:05 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-03-10 11:05:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_651\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_652\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_165\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNote that we're not running the Swift daemon that consumes that header and deletes objects yet (swift-object-expirer), afaik. It's available in Debian.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_658\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6012398\" id=\"6012398\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user phui-timeline-icon\" data-meta=\"0_657\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_166\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e reassigned this task from \u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e to \u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6012398\" data-sigil=\"has-tooltip\" data-meta=\"0_656\"\u003e\u003cspan class=\"screen-only\"\u003eMar 30 2020, 7:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-03-30 19:56:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_667\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6211680\" id=\"6211680\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_666\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_167\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6211680\" data-sigil=\"has-tooltip\" data-meta=\"0_665\"\u003e\u003cspan class=\"screen-only\"\u003eJun 10 2020, 3:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-10 15:54:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_663\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_664\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_168\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 489022 had a related patch set uploaded (by Gilles; owner: Gilles):\u003cbr \/\u003e\n[operations\/software\/thumbor-plugins@master] Set expiry headers on thumbnails\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/489022\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/489022\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_670\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ja2jvvk7rqgjkczqi4sv\/PHID-FILE-5elxaqu5owwyq2igmilb\/alphanumeric_lato-white_A.png-_3ba984-0%2C0%2C0%2C0.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/alistair3149\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6229134\" id=\"6229134\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_669\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/alistair3149\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_169\"\u003ealistair3149\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6229134\" data-sigil=\"has-tooltip\" data-meta=\"0_668\"\u003e\u003cspan class=\"screen-only\"\u003eJun 16 2020, 6:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-06-16 18:29:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_673\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ac3l7j336ju4mdgbssjt\/PHID-FILE-iewutaglmkyrs7tcmtr7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/jijiki\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6442153\" id=\"6442153\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_672\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/jijiki\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_170\"\u003ejijiki\u003c\/a\u003e removed a project: \u003ca href=\"\/tag\/user-jijiki\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_171\"\u003eUser-jijiki\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6442153\" data-sigil=\"has-tooltip\" data-meta=\"0_671\"\u003e\u003cspan class=\"screen-only\"\u003eSep 8 2020, 10:20 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-09-08 10:20:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_676\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/qdwewbx3e7yxhxche6ch\/PHID-FILE-522xul437qimyz7aedqw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dpifke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6579595\" id=\"6579595\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_675\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_172\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_173\"\u003eBlocked (old)\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_174\"\u003eDoing (old)\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_175\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6579595\" data-sigil=\"has-tooltip\" data-meta=\"0_674\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2020, 6:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-26 18:39:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_685\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/qdwewbx3e7yxhxche6ch\/PHID-FILE-522xul437qimyz7aedqw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dpifke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6579630\" id=\"6579630\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_684\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_176\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6579630\" data-sigil=\"has-tooltip\" data-meta=\"0_683\"\u003e\u003cspan class=\"screen-only\"\u003eOct 26 2020, 6:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-26 18:42:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_681\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_682\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe Swift object-expirer is running in beta if we want to start testing this there.\u003c\/p\u003e\n\n\u003cp\u003eThere are some loose ends before it's enabled in prod, but it doesn't hurt to set the header even if it isn't yet being acted on.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_694\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"6580669\" id=\"6580669\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_693\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_178\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6580669\" data-sigil=\"has-tooltip\" data-meta=\"0_692\"\u003e\u003cspan class=\"screen-only\"\u003eOct 27 2020, 8:33 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-10-27 08:33:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_690\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_691\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_179\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFrom my POV as Swift maintainer I'm ok to go ahead with testing etc.\u003c\/p\u003e\n\n\u003cp\u003eAlthough it'd be really good to have at least stats(d) for the expirer in prod to monitor activity (i.e. \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T229584#6516633\" class=\"phui-tag-view phui-tag-type-shade phui-tag-blue phui-tag-shade phui-tag-icon-view \" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-anchor\" data-meta=\"0_33\" aria-hidden=\"true\"\u003e\u003c\/span\u003ehttps:\/\/phabricator.wikimedia.org\/T229584#6516633\u003c\/span\u003e\u003c\/a\u003e) has there been any action on that \u003ca href=\"\/p\/dpifke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@dpifke\u003c\/span\u003e\u003c\/a\u003e ?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_697\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/3uc2yc67cpe2vkc4sibj\/PHID-FILE-5czt7j26qbkmfpxtwswx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Peachey88\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6714556\" id=\"6714556\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_696\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Peachey88\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_180\"\u003ePeachey88\u003c\/a\u003e mentioned this in \u003ca href=\"\/T269946\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_181\"\u003eT269946: Enable webp thumbnails on all images for non-Commons wikis\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6714556\" data-sigil=\"has-tooltip\" data-meta=\"0_695\"\u003e\u003cspan class=\"screen-only\"\u003eDec 30 2020, 2:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2020-12-30 14:29:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_700\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/giz4kiiwghycxmhpfwnm\/PHID-FILE-hoikbdz5d2t6fnwo7yxy\/bbbc81-alphanumeric_lato-dark_L.png-255%2C255%2C255%2C0.4.png)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/LSobanski\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6722366\" id=\"6722366\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_699\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/LSobanski\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_182\"\u003eLSobanski\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6722366\" data-sigil=\"has-tooltip\" data-meta=\"0_698\"\u003e\u003cspan class=\"screen-only\"\u003eJan 5 2021, 1:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-05 13:35:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_703\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/skjcbgvgg73sopweegum\/PHID-FILE-5fawjhav2batqyi4ixtx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ArielGlenn\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6735284\" id=\"6735284\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_702\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ArielGlenn\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_183\"\u003eArielGlenn\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6735284\" data-sigil=\"has-tooltip\" data-meta=\"0_701\"\u003e\u003cspan class=\"screen-only\"\u003eJan 11 2021, 9:35 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-01-11 09:35:23 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_706\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/m25d4sm73mzvfxk7efxx\/PHID-FILE-zigk2s47ziyr7hqvpw7v\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Ladsgroup\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6829421\" id=\"6829421\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_705\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Ladsgroup\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_184\"\u003eLadsgroup\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6829421\" data-sigil=\"has-tooltip\" data-meta=\"0_704\"\u003e\u003cspan class=\"screen-only\"\u003eFeb 14 2021, 10:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-02-14 22:10:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_709\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6876413\" id=\"6876413\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_708\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_185\"\u003eKrinkle\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_186\"\u003eDoing (old)\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_187\"\u003eInbox, needs triage\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_188\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6876413\" data-sigil=\"has-tooltip\" data-meta=\"0_707\"\u003e\u003cspan class=\"screen-only\"\u003eMar 2 2021, 9:08 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-02 21:08:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_712\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"6876664\" id=\"6876664\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_711\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_189\"\u003eKrinkle\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_190\"\u003eInbox, needs triage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003eBacklog: Maintenance, non-prioritized\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_192\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#6876664\" data-sigil=\"has-tooltip\" data-meta=\"0_710\"\u003e\u003cspan class=\"screen-only\"\u003eMar 2 2021, 9:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-03-02 21:33:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_715\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7185106\" id=\"7185106\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_714\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_193\"\u003efgiunchedi\u003c\/a\u003e mentioned this in \u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_194\"\u003eUnknown Object (Task)\u003c\/span\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7185106\" data-sigil=\"has-tooltip\" data-meta=\"0_713\"\u003e\u003cspan class=\"screen-only\"\u003eJun 29 2021, 8:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-06-29 20:30:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_725\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7226721\" id=\"7226721\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_724\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_195\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#7226721\" data-sigil=\"has-tooltip\" data-meta=\"0_723\"\u003e\u003cspan class=\"screen-only\"\u003eJul 21 2021, 9:30 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-07-21 09:30:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_721\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_722\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIn light of longer than expected lead time for new ms-be hardware (T284953) I'd like to explore again the object expirer road which is IMHO the proper solution in general. Is there anything I can do to help and\/or move the needle? What else needs to happen? Thank you !\u003c\/p\u003e\n\n\u003cp\u003eedit: if lead times for this are long and\/or too complex that's fine too for now, I'd like to get an idea of what options from \u003ca href=\"https:\/\/phabricator.wikimedia.org\/T284953#7185103\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/phabricator.wikimedia.org\/T284953#7185103\u003c\/a\u003e we can realistically do\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_734\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/qdwewbx3e7yxhxche6ch\/PHID-FILE-522xul437qimyz7aedqw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dpifke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7227693\" id=\"7227693\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_733\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_197\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7227693\" data-sigil=\"has-tooltip\" data-meta=\"0_732\"\u003e\u003cspan class=\"screen-only\"\u003eJul 21 2021, 3:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-07-21 15:46:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_730\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_731\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSorry, this kinda dropped off my radar.\u003c\/p\u003e\n\n\u003cp\u003eThe object-expirer works (it's been running in deployment-prep with no issues for quite a while now) - we just need to turn it on in prod, with a prerequisite of adding monitoring for it first.\u003c\/p\u003e\n\n\u003cp\u003eI'll look later today or tomorrow at what's involved in getting stats out of it.  Hopefully it's pretty straightforward.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_743\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/qdwewbx3e7yxhxche6ch\/PHID-FILE-522xul437qimyz7aedqw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/dpifke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7227857\" id=\"7227857\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_742\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/dpifke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_199\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e dpifke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7227857\" data-sigil=\"has-tooltip\" data-meta=\"0_741\"\u003e\u003cspan class=\"screen-only\"\u003eJul 21 2021, 4:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-07-21 16:24:00 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_739\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_740\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_200\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eLooks like we're already tracking DELETEs, e.g. the second panel in \u003ca href=\"https:\/\/grafana-rw.wikimedia.org\/d\/OPgmB1Eiz\/swift?orgId=1\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana-rw.wikimedia.org\/d\/OPgmB1Eiz\/swift?orgId=1\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cp\u003eDo you think other stats are needed?  Should we create an alert if the ratio of deletes to other operations exceeds some threshold?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_752\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"7235439\" id=\"7235439\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_751\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_201\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7235439\" data-sigil=\"has-tooltip\" data-meta=\"0_750\"\u003e\u003cspan class=\"screen-only\"\u003eJul 26 2021, 8:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-07-26 08:40:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_748\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_749\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T211661#7227857\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT211661#7227857\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/dpifke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@dpifke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eLooks like we're already tracking DELETEs, e.g. the second panel in \u003ca href=\"https:\/\/grafana-rw.wikimedia.org\/d\/OPgmB1Eiz\/swift?orgId=1\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/grafana-rw.wikimedia.org\/d\/OPgmB1Eiz\/swift?orgId=1\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cp\u003eDo you think other stats are needed?  Should we create an alert if the ratio of deletes to other operations exceeds some threshold?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWhat I was referring to is operations by the expirer itself, the panel in the dashboard is overall DELETEs.  Certainly an alert for "high deletes" of some kind might be useful regardless, good idea!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_755\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7308976\" id=\"7308976\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_754\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_203\"\u003efgiunchedi\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/76\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_204\"\u003eInbox\u003c\/a\u003e to \u003ca href=\"\/project\/board\/76\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_205\"\u003eRadar\u003c\/a\u003e on the \u003ca href=\"\/tag\/sre-swift-storage\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_206\"\u003eSRE-swift-storage\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7308976\" data-sigil=\"has-tooltip\" data-meta=\"0_753\"\u003e\u003cspan class=\"screen-only\"\u003eAug 25 2021, 4:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-08-25 16:19:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_758\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"7412948\" id=\"7412948\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_757\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_207\"\u003eKrinkle\u003c\/a\u003e added a parent task: \u003ca href=\"\/T269946\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_208\"\u003eT269946: Enable webp thumbnails on all images for non-Commons wikis\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#7412948\" data-sigil=\"has-tooltip\" data-meta=\"0_756\"\u003e\u003cspan class=\"screen-only\"\u003eOct 8 2021, 5:48 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2021-10-08 17:48:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-spcngbyufhlv6qsdoqev"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-s67l67gutsve6hxuigsu"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-nddfwrtxjliyzpyi425w"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ctafnlifn64axvokqjy3"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-qwbirpxueu2426ul2odw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-veort6imbjquv3sg33vl"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},[],{"hovercardSpec":{"objectPHID":"PHID-TASK-eguf6f6gk3rwrzyb2lws"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg","contextPHID":"PHID-TASK-q7rcpaxadgsevowgok22"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jzttt2rzt4bpbzkerkod"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ylarhsvhxcxqbur6pbmv"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-mzxlpm2u56pgpkqn3onq"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-oezmppzx6oodxhitma2m"}},{"hovercardSpec":{"objectPHID":"PHID-USER-xragmpydvtwulk3zv4o7"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-c4yrc5wlq7aalby54odc"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-knvdos3w6r5kdmycbgir"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-rqrbozlyfvdfdcv"},{"hovercardSpec":{"objectPHID":"PHID-USER-6b6w5qogf4deywgbkuzs"}},{"phid":"PHID-XACT-TASK-kwnacsek64ahfut"},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-pwio6mw7vkh6aoq"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-ilsup6tc77243oo"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-w2syv2qx3qbgfaz"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-uh2tfjwg46n4bym"},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-5iaicgogin6jtik"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-lhtlnmkdbzlz6pbxaqdd"}},{"phid":"PHID-XACT-TASK-swxpz62xixf7zmq"},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-3kgo6g27whfrnpj"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-5gnw3jgslkvjd4c"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-4drrayh3tvnxgvj"},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-yej2ce6vyvwez3n"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-m7w4xrtpkx7dy4c"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-4d7ey3fvcusutk3"},{"hovercardSpec":{"objectPHID":"PHID-USER-abszjqutasfjxgagymds"}},{"hovercardSpec":{"objectPHID":"PHID-USER-m67fypflxledchg2mst7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-ngvo53qe7tntm7z"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-fogq3mirsf7w6iw"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"phid":"PHID-XACT-TASK-lpki42y3acgouom"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-eiigr7hbppc53m7"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"phid":"PHID-XACT-TASK-7itxcrls7pwwtfe"},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-audv5c3bg7qcyth"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-prrfijjpzvkstcw"},{"phid":"PHID-XACT-TASK-i4wxaowr6ulorpe"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"phid":"PHID-XACT-TASK-hvuaizi2rnec7sx"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-oezksamlw52dmdeufny6"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-7xng6yzpisso3g5fvamm"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hfqxkl5wvmdkecl5utfy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-wudwruqnwwjanhetyvaf"}},{"phid":"PHID-XACT-TASK-qb2cpz64jbqtvs2"},{"hovercardSpec":{"objectPHID":"PHID-USER-bmbmvb2e5bg6ztcangbq"}},{"hovercardSpec":{"objectPHID":"PHID-USER-v7vgzvvcw7v2umf737ri"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-7revoxn7kere4ix"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ca6m35hviot32dyabyoy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-2hemjyy2n3gcfq7"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-CMIT-moevz6gt4nwuxlug366n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-mzxlpm2u56pgpkqn3onq"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-3mtagnxouqrnrz6xtfta"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-7p7xfhzjwgewnri"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-qebao4qsaeftmup"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"phid":"PHID-XACT-TASK-ojbobghdljyak62"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-jfkcg3vhb77depn"},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-l7i5j2feyc5mnrmmxnjg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-dsnetxv4ht3ml4w"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-uhf3oh27aqeibvv"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-ur32yjp2hmmehrt"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-veort6imbjquv3sg33vl"}},{"phid":"PHID-XACT-TASK-kpiwyhaseenga73"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-22huwrywjnolgxy"},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-oezmppzx6oodxhitma2m"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-3mtagnxouqrnrz6xtfta"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-iflzuashmah3acqgc3yc"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-axdirzlqgxa7sny"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-y42mmlwjj667s56"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg"}},{"phid":"PHID-XACT-TASK-7425ho753obah5g"},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg"}},{"phid":"PHID-XACT-TASK-l3wjvi4cldkzohu"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"phid":"PHID-XACT-TASK-mgwbo6ptbunyvwj"},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-5gtr2ghwy3gy6rs"},{"hovercardSpec":{"objectPHID":"PHID-USER-nzterrrjhsabuvagdv2x"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dp5h2v3xwdzdh4dug36u"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-l7i5j2feyc5mnrmmxnjg"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-iflzuashmah3acqgc3yc"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-5tymc6r6xqlaota445ou"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg"}},{"phid":"PHID-XACT-TASK-7xfqle3wv4x5msx"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-v3kuak2t7g5dvx6"},{"hovercardSpec":{"objectPHID":"PHID-USER-hwuumsjqciw7ji5qhpc7"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-hi5rnsgqztvcqewqmpdy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-eo4cqx5xwnf4ojlk4bbl"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jtxavgb3caz53o45csni"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ctchvhisqfvk4rz7nt7d"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-5tymc6r6xqlaota445ou"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ylarhsvhxcxqbur6pbmv"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ylarhsvhxcxqbur6pbmv"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-4hnpc5ihvpxbag7deejp"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-oriw6y4sfyk6tghpyfdu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-f6xwg3c4axzjfaq"},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg"}},{"phid":"PHID-XACT-TASK-3fkp34hh42y4gp5"},{"hovercardSpec":{"objectPHID":"PHID-USER-tircs6eaqb2i6ib3rweg"}},{"phid":"PHID-XACT-TASK-lgiuawemsalevfn"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-7oqdnc3g6v4vjka"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-nsydk3pvrm6tg5gmd52x"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-kw25kh5b5fqxv5agb4r3"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-iodggnykzbdrhsjfkg4q"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-hi5rnsgqztvcqewqmpdy"}},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-g3p4rm6f76f7nsk","anchor":"4816882"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-f2mtdvtg5neogup","anchor":"4829175"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3uvmzki74rcjxkr","anchor":"4836858"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-a6oa7noeuk4kdze","anchor":"4912791"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rqrbozlyfvdfdcv\/","ref":"T211661#4926909"},[],{"anchor":"4926909"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_221\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_222\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rqrbozlyfvdfdcv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_223\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_224\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rqrbozlyfvdfdcv","anchor":"4926909"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jb6mkhvt47je6sp","anchor":"4927888"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kwnacsek64ahfut\/","ref":"T211661#4928551"},[],{"anchor":"4928551"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_233\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kwnacsek64ahfut\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_235\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-kwnacsek64ahfut","anchor":"4928551"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pwio6mw7vkh6aoq\/","ref":"T211661#4928637"},[],{"anchor":"4928637"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_244\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pwio6mw7vkh6aoq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_246\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-pwio6mw7vkh6aoq","anchor":"4928637"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ilsup6tc77243oo\/","ref":"T211661#4928797"},[],{"anchor":"4928797"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_255\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ilsup6tc77243oo\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_257\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_258\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ilsup6tc77243oo","anchor":"4928797"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-w2syv2qx3qbgfaz\/","ref":"T211661#4928804"},[],{"anchor":"4928804"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_264\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-w2syv2qx3qbgfaz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_266\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-w2syv2qx3qbgfaz","anchor":"4928804"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uh2tfjwg46n4bym\/","ref":"T211661#4928838"},[],{"anchor":"4928838"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_273\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uh2tfjwg46n4bym\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_275\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uh2tfjwg46n4bym","anchor":"4928838"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5iaicgogin6jtik\/","ref":"T211661#4928875"},[],{"anchor":"4928875"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_282\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5iaicgogin6jtik\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_284\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-5iaicgogin6jtik","anchor":"4928875"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-swxpz62xixf7zmq\/","ref":"T211661#4928884"},[],{"anchor":"4928884"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_293\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-swxpz62xixf7zmq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_295\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-onnp46ep6p4yrhb","anchor":"4928884"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3kgo6g27whfrnpj\/","ref":"T211661#4929122"},[],{"anchor":"4929122"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_302\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3kgo6g27whfrnpj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_304\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3kgo6g27whfrnpj","anchor":"4929122"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5gnw3jgslkvjd4c\/","ref":"T211661#4930751"},[],{"anchor":"4930751"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_313\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5gnw3jgslkvjd4c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_315\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_316\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5gnw3jgslkvjd4c","anchor":"4930751"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4drrayh3tvnxgvj\/","ref":"T211661#4931056"},[],{"anchor":"4931056"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_322\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4drrayh3tvnxgvj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_324\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_325\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4drrayh3tvnxgvj","anchor":"4931056"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yej2ce6vyvwez3n\/","ref":"T211661#4931840"},[],{"anchor":"4931840"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_331\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yej2ce6vyvwez3n\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_333\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_334\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-yej2ce6vyvwez3n","anchor":"4931840"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u3opzcxxyojcfbq","anchor":"4931971"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-m7w4xrtpkx7dy4c\/","ref":"T211661#4932147"},[],{"anchor":"4932147"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_345\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-m7w4xrtpkx7dy4c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_347\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_348\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-m7w4xrtpkx7dy4c","anchor":"4932147"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4d7ey3fvcusutk3\/","ref":"T211661#4932156"},[],{"anchor":"4932156"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_354\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_355\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4d7ey3fvcusutk3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_356\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_357\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-4d7ey3fvcusutk3\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_358\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4d7ey3fvcusutk3","anchor":"4932156"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2habde5k6ws7hvh","anchor":"4932257"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rxzyidgxxhxb3yu","anchor":"4932258"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ngvo53qe7tntm7z\/","ref":"T211661#4934095"},[],{"anchor":"4934095"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_370\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ngvo53qe7tntm7z\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_372\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_373\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ngvo53qe7tntm7z","anchor":"4934095"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fogq3mirsf7w6iw\/","ref":"T211661#4934161"},[],{"anchor":"4934161"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_379\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_380\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fogq3mirsf7w6iw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_381\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_382\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fogq3mirsf7w6iw","anchor":"4934161"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lpki42y3acgouom\/","ref":"T211661#4934164"},[],{"anchor":"4934164"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_388\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_389\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lpki42y3acgouom\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_390\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_391\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lpki42y3acgouom","anchor":"4934164"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-eiigr7hbppc53m7\/","ref":"T211661#4934183"},[],{"anchor":"4934183"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_397\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_398\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eiigr7hbppc53m7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_399\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_400\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eiigr7hbppc53m7","anchor":"4934183"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7itxcrls7pwwtfe\/","ref":"T211661#4934261"},[],{"anchor":"4934261"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_406\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_407\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7itxcrls7pwwtfe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_408\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_409\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7itxcrls7pwwtfe","anchor":"4934261"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-audv5c3bg7qcyth\/","ref":"T211661#4935167"},[],{"anchor":"4935167"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_415\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_416\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-audv5c3bg7qcyth\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_417\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_418\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-audv5c3bg7qcyth\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_419\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-audv5c3bg7qcyth","anchor":"4935167"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-prrfijjpzvkstcw\/","ref":"T211661#4936243"},[],{"anchor":"4936243"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_427\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_428\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-prrfijjpzvkstcw\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_429\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_430\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-prrfijjpzvkstcw\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-prrfijjpzvkstcw","anchor":"4936243"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-i4wxaowr6ulorpe\/","ref":"T211661#4936372"},[],{"anchor":"4936372"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_437\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_438\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-i4wxaowr6ulorpe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_439\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-i4wxaowr6ulorpe","anchor":"4936372"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hvuaizi2rnec7sx\/","ref":"T211661#4936373"},[],{"anchor":"4936373"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_446\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_447\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hvuaizi2rnec7sx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_448\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2zd23pxsrwoio6p","anchor":"4936373"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-wr3qz4ugtuzfbfi","anchor":"4936427"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-l2fod6dl7jfjyi2","anchor":"4944275"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6rbodkdtvpsx34a","anchor":"4945314"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-wajpzei6xkwhpr4","anchor":"4950335"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qb2cpz64jbqtvs2\/","ref":"T211661#4959116"},[],{"anchor":"4959116"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_467\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_468\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qb2cpz64jbqtvs2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_469\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_470\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qb2cpz64jbqtvs2","anchor":"4959116"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gfcsv33jl6ys5uq","anchor":"4959157"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7revoxn7kere4ix\/","ref":"T211661#4959380"},[],{"anchor":"4959380"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_479\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_480\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7revoxn7kere4ix\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_481\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_482\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-7revoxn7kere4ix\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_483\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7revoxn7kere4ix","anchor":"4959380"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3xgxfy3zdhq4g5t","anchor":"5015235"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2hemjyy2n3gcfq7\/","ref":"T211661#5025597"},[],{"anchor":"5025597"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_492\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_493\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2hemjyy2n3gcfq7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_494\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_495\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2hemjyy2n3gcfq7","anchor":"5025597"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-b7iyioliazr6tn4","anchor":"5043052"},{"tip":"Via Daemon"},[],{"phid":"PHID-XACT-TASK-gbzkcfymf76k3jl","anchor":"5089823"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-aadi3kzzdcwdcdq","anchor":"5132317"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gv3lstj3u7ife3n","anchor":"5268647"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7p7xfhzjwgewnri\/","ref":"T211661#5273185"},[],{"anchor":"5273185"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_514\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_515\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7p7xfhzjwgewnri\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_516\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_517\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-ghzjrhansmf53ti","anchor":"5273185"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-qebao4qsaeftmup\/","ref":"T211661#5273190"},[],{"anchor":"5273190"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_524\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_525\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-qebao4qsaeftmup\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_526\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_527\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qebao4qsaeftmup","anchor":"5273190"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ojbobghdljyak62\/","ref":"T211661#5273209"},[],{"anchor":"5273209"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_533\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_534\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ojbobghdljyak62\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_535\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_536\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ojbobghdljyak62","anchor":"5273209"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jfkcg3vhb77depn\/","ref":"T211661#5273217"},[],{"anchor":"5273217"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_542\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_543\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jfkcg3vhb77depn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_544\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_545\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jfkcg3vhb77depn","anchor":"5273217"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-qod5xf7wiojq5ev","anchor":"5273218"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dsnetxv4ht3ml4w\/","ref":"T211661#5273224"},[],{"anchor":"5273224"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_554\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_555\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dsnetxv4ht3ml4w\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_556\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_557\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dsnetxv4ht3ml4w","anchor":"5273224"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uhf3oh27aqeibvv\/","ref":"T211661#5273370"},[],{"anchor":"5273370"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_563\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uhf3oh27aqeibvv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_565\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uhf3oh27aqeibvv","anchor":"5273370"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-salqaumsdchthb5","anchor":"5273427"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-w5p3ju2mc6gvug4","anchor":"5273949"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ur32yjp2hmmehrt\/","ref":"T211661#5288525"},[],{"anchor":"5288525"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_578\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_579\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ur32yjp2hmmehrt\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_580\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_581\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ur32yjp2hmmehrt","anchor":"5288525"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kpiwyhaseenga73\/","ref":"T211661#5288592"},[],{"anchor":"5288592"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_587\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_588\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kpiwyhaseenga73\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_589\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_590\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-uade3ftky5pqpf5","anchor":"5288592"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-22huwrywjnolgxy\/","ref":"T211661#5288667"},[],{"anchor":"5288667"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_596\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_597\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-22huwrywjnolgxy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_598\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_599\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-22huwrywjnolgxy","anchor":"5288667"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-fnjeiyomzhjfvca","anchor":"5297943"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ctyqnbqv2askqsx","anchor":"5535831"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-axdirzlqgxa7sny\/","ref":"T211661#5681381"},[],{"anchor":"5681381"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_611\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_612\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-axdirzlqgxa7sny\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_613\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_614\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-axdirzlqgxa7sny","anchor":"5681381"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-y42mmlwjj667s56\/","ref":"T211661#5770405"},[],{"anchor":"5770405"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_620\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_621\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-y42mmlwjj667s56\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_622\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_623\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-y42mmlwjj667s56","anchor":"5770405"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7425ho753obah5g\/","ref":"T211661#5953594"},[],{"anchor":"5953594"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_629\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_630\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7425ho753obah5g\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_631\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_632\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-doaxjk5utskv4aq","anchor":"5953594"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-l3wjvi4cldkzohu\/","ref":"T211661#5954991"},[],{"anchor":"5954991"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_638\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_639\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-l3wjvi4cldkzohu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_640\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_641\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-l3wjvi4cldkzohu","anchor":"5954991"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mgwbo6ptbunyvwj\/","ref":"T211661#5955660"},[],{"anchor":"5955660"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_647\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_648\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mgwbo6ptbunyvwj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_649\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_650\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mgwbo6ptbunyvwj","anchor":"5955660"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-adhtfta7sgnst5z","anchor":"6012398"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5gtr2ghwy3gy6rs\/","ref":"T211661#6211680"},[],{"anchor":"6211680"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_659\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_660\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5gtr2ghwy3gy6rs\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_661\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_662\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5gtr2ghwy3gy6rs","anchor":"6211680"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ihejzplil5rhr2a","anchor":"6229134"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6fvxdlcnqqpzgu7","anchor":"6442153"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ligxq36symjvp74","anchor":"6579595"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7xfqle3wv4x5msx\/","ref":"T211661#6579630"},[],{"anchor":"6579630"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_677\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_678\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7xfqle3wv4x5msx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_679\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_680\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7xfqle3wv4x5msx","anchor":"6579630"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-v3kuak2t7g5dvx6\/","ref":"T211661#6580669"},[],{"anchor":"6580669"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_686\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_687\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v3kuak2t7g5dvx6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_688\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_689\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-v3kuak2t7g5dvx6","anchor":"6580669"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mzxzbs4d7hofrtx","anchor":"6714556"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ctijkhjhcyufylk","anchor":"6722366"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ax2vxo5t7hmftuo","anchor":"6735284"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dnuamtd74hbmq7o","anchor":"6829421"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-574mv377wkue22h","anchor":"6876413"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ugvz3yd2tj3nz7d","anchor":"6876664"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u4uncigwfbvz27m","anchor":"7185106"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-f6xwg3c4axzjfaq\/","ref":"T211661#7226721"},[],{"anchor":"7226721"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_716\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_717\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-f6xwg3c4axzjfaq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_718\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_719\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-f6xwg3c4axzjfaq\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_720\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-f6xwg3c4axzjfaq","anchor":"7226721"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3fkp34hh42y4gp5\/","ref":"T211661#7227693"},[],{"anchor":"7227693"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_726\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_727\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3fkp34hh42y4gp5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_728\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_729\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3fkp34hh42y4gp5","anchor":"7227693"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lgiuawemsalevfn\/","ref":"T211661#7227857"},[],{"anchor":"7227857"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_735\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_736\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lgiuawemsalevfn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_737\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_738\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lgiuawemsalevfn","anchor":"7227857"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7oqdnc3g6v4vjka\/","ref":"T211661#7235439"},[],{"anchor":"7235439"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_744\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_745\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7oqdnc3g6v4vjka\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_746\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_747\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7oqdnc3g6v4vjka","anchor":"7235439"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eha4nepbfd2cr3g","anchor":"7308976"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-c3frlsltgzieomy","anchor":"7412948"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/cc8a2c6b\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}