for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3582425\" id=\"3582425\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_49\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003eKrinkle\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3582425\" data-sigil=\"has-tooltip\" data-meta=\"0_48\"\u003e\u003cspan class=\"screen-only\"\u003eSep 5 2017, 10:25 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-05 22:25:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3582515\" id=\"3582515\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_52\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003eKrinkle\u003c\/a\u003e mentioned this in \u003ca href=\"\/T158837\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003eT158837: Consolidate performance website and related software\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3582515\" data-sigil=\"has-tooltip\" data-meta=\"0_51\"\u003e\u003cspan class=\"screen-only\"\u003eSep 5 2017, 10:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-05 22:42:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_56\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3582604\" id=\"3582604\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_55\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003eKrinkle\u003c\/a\u003e added a parent task: \u003ca href=\"\/T110903\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003eT110903: Make webperf eventlogging consumers use eventlogging on Kafka\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3582604\" data-sigil=\"has-tooltip\" data-meta=\"0_54\"\u003e\u003cspan class=\"screen-only\"\u003eSep 5 2017, 11:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-05 23:01:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_60\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3583322\" id=\"3583322\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-trophy phui-timeline-icon\" data-meta=\"0_58\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view sprite-tokens tokens-heart-1 phui-timeline-token\" data-meta=\"0_59\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003efgiunchedi\u003c\/a\u003e awarded a token.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3583322\" data-sigil=\"has-tooltip\" data-meta=\"0_57\"\u003e\u003cspan class=\"screen-only\"\u003eSep 6 2017, 8:07 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-06 08:07:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3583327\" id=\"3583327\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_68\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3583327\" data-sigil=\"has-tooltip\" data-meta=\"0_67\"\u003e\u003cspan class=\"screen-only\"\u003eSep 6 2017, 8:14 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-06 08:14:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_65\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_66\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_15\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThanks \u003ca href=\"\/p\/Krinkle\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Krinkle\u003c\/span\u003e\u003c\/a\u003e for starting this!\u003c\/p\u003e\n\n\u003cp\u003eImplementation wise it should be easy to add the Prometheus python client to existing daemons and open up ports where metrics will be scraped by Prometheus, while keeping existing statsd\/coal metrics during the transition.\u003c\/p\u003e\n\n\u003cp\u003eRetention wise, we're running several Prometheus instances with different retention policies. The \u003ctt class=\"remarkup-monospaced\"\u003eglobal\u003c\/tt\u003e instance aggregates data from all other instances and has ATM one year retention, though I believe we can reasonably extend it even further since it is essentially bound by how much disk space and memory there is available.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_72\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/st3g3yziltbwjorfzd75\/PHID-FILE-d5zes5mjcqa2dfoswe23\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Peter\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3584744\" id=\"3584744\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_71\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Peter\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003ePeter\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3584744\" data-sigil=\"has-tooltip\" data-meta=\"0_70\"\u003e\u003cspan class=\"screen-only\"\u003eSep 6 2017, 3:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-06 15:43:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/abnv6m2xoqjroqyladvh\/PHID-FILE-yryhnwq73gr36rjgrz4m\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Gilles\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3599711\" id=\"3599711\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_74\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Gilles\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Gilles\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003eInbox, needs triage\u003c\/a\u003e to \u003ca href=\"\/project\/board\/1212\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003eTo-do: Goals, prioritized next 4 Quarters\u003c\/a\u003e on the \u003ca href=\"\/tag\/performance-team\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003ePerformance-Team\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3599711\" data-sigil=\"has-tooltip\" data-meta=\"0_73\"\u003e\u003cspan class=\"screen-only\"\u003eSep 12 2017, 9:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-12 09:40:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_78\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3599742\" id=\"3599742\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_77\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003eKrinkle\u003c\/a\u003e added a parent task: \u003ca href=\"\/T159354\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003eT159354: Move coal from graphite#001 nodes to webperf#001\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3599742\" data-sigil=\"has-tooltip\" data-meta=\"0_76\"\u003e\u003cspan class=\"screen-only\"\u003eSep 12 2017, 9:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-12 09:46:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/st3g3yziltbwjorfzd75\/PHID-FILE-d5zes5mjcqa2dfoswe23\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Peter\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3640903\" id=\"3640903\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-trophy phui-timeline-icon\" data-meta=\"0_80\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view sprite-tokens tokens-heart-1 phui-timeline-token\" data-meta=\"0_81\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Peter\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003ePeter\u003c\/a\u003e awarded a token.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3640903\" data-sigil=\"has-tooltip\" data-meta=\"0_79\"\u003e\u003cspan class=\"screen-only\"\u003eSep 27 2017, 7:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-09-27 19:50:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3775948\" id=\"3775948\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_84\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003eKrinkle\u003c\/a\u003e mentioned this in \u003ca href=\"\/T110903\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_25\"\u003eT110903: Make webperf eventlogging consumers use eventlogging on Kafka\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3775948\" data-sigil=\"has-tooltip\" data-meta=\"0_83\"\u003e\u003cspan class=\"screen-only\"\u003eNov 20 2017, 9:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-11-20 21:29:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_85\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003eKrinkle\u003c\/a\u003e removed a parent task: \u003ca href=\"\/T110903\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_27\"\u003eT110903: Make webperf eventlogging consumers use eventlogging on Kafka\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" aria-label=\"Continuous Integrator\" data-sigil=\"has-tooltip\" data-meta=\"0_95\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_96\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3776310\" id=\"3776310\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eKrinkle\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3776310\" data-sigil=\"has-tooltip\" data-meta=\"0_93\"\u003e\u003cspan class=\"screen-only\"\u003eNov 20 2017, 11:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-11-20 23:20:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_91\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_92\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_29\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e I'm looking into Prometheus and have got a couple of questions.\u003c\/p\u003e\n\n\u003cp\u003eThe end result I'd like to aim for is to have our \u00b1 10 metrics from Navigation Timing available, through Prometheus, in Grafana with the following capabilities:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eProduce a single value representing the median, p75, or p95 over a given period of time (e.g. p95 of last month).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eProduce a series of values representing the median, p75 or p95 per given interval (e.g. per 15min, per hour or per day).\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eProduce a histogram or heatmap that considers all original values.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe previous 3 use cases, but filtered by labels with limited pre-defined values such as browser family and country code.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eThese first two cases we try currently via Graphite, but this fails somewhat because the raw data is lost. So we end up averaging per-minute percentiles (which is horrible). Even if we change the interval for NavTiming in Graphite to per-second instead of per-minute, this would make the data not statistically meaningful. We could improve the current situation by manually aggregating into manually-crafted buckets, but it seems Prometheus would be a better option.\u003c\/p\u003e\n\n\u003cp\u003eHowever, I'm somewhat unclear on how we should use Prometheus.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003e\u003cstrong\u003eLabels\u003c\/strong\u003e \u003ca href=\"https:\/\/prometheus.io\/docs\/practices\/instrumentation\/#do-not-overuse-labels\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/practices\/instrumentation\/#do-not-overuse-labels\u003c\/a\u003e\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAs a general guideline, try to keep the cardinality of your metrics below 10, and for metrics that exceed that, aim to limit them to a handful across your whole system. The vast majority of your metrics should have no labels.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eOur metrics currently have five labels:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eplatform (2 values: mobile, desktop)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eauth (2 values: anonymous, authenticated)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ebrowser (one of < 12 handpicked browser names, or "Other")\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econtinent (one of 6 values, or null)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecountry (one of 8 values, or null)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eWould that be fine?\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003e\u003cstrong\u003eMetric type\u003c\/strong\u003e\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"https:\/\/prometheus.io\/docs\/concepts\/metric_types\/#gauge\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/concepts\/metric_types\/\u003c\/a\u003e\u003cbr \/\u003e\nGauge\u003cbr \/\u003e\n\u003ca href=\"https:\/\/prometheus.io\/docs\/practices\/histograms\/\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/practices\/histograms\/\u003c\/a\u003e\u003cbr \/\u003e\nHistograms and summaries\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIt sounds like Summaries is similar to what we do now \u2013 calculate the percentiles per-minute in StatsD (before sending to Graphite) at which points it becomes a single aggregated value, not appropiate for further aggregation.\u003c\/p\u003e\n\n\u003cp\u003eHistograms seem attractive given the query flexibility after the fact. Although the warning "not aggregatable" only applying to Summaries on that page, it sounds to me like it should apply to Histogram as well, given that it will not accurately tell you the median over 3 days in a way that considers all original values. It can accurately say in which bucket the value is (which is an improvement over our current Statsd\/Graphite approach), but it not being able to produce an actual number makes it hard to use for our use case, which is to catch small regressions in individual features and products, not just whether we're still meeting an SLA or not.\u003c\/p\u003e\n\n\u003cp\u003eGiven Prometheus' approach to not lose original input data, I'm leaning towards using a Gauge. Assuming the "we don't aggregate" principle applies there, that would mean all original data points are preserved and could be queried\/aggregated upon request. Although naturally I would imagine cannot be very good for query performance. Right?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_106\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3777688\" id=\"3777688\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_105\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3777688\" data-sigil=\"has-tooltip\" data-meta=\"0_104\"\u003e\u003cspan class=\"screen-only\"\u003eNov 21 2017, 12:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-11-21 12:46:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_102\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_103\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T175087#3776310\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT175087#3776310\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Krinkle\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Krinkle\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eHowever, I'm somewhat unclear on how we should use Prometheus.\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003e\u003cstrong\u003eLabels\u003c\/strong\u003e \u003ca href=\"https:\/\/prometheus.io\/docs\/practices\/instrumentation\/#do-not-overuse-labels\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/practices\/instrumentation\/#do-not-overuse-labels\u003c\/a\u003e\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAs a general guideline, try to keep the cardinality of your metrics below 10, and for metrics that exceed that, aim to limit them to a handful across your whole system. The vast majority of your metrics should have no labels.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eOur metrics currently have five labels:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eplatform (2 values: mobile, desktop)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eauth (2 values: anonymous, authenticated)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ebrowser (one of < 12 handpicked browser names, or "Other")\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003econtinent (one of 6 values, or null)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ecountry (one of 8 values, or null)\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eWould that be fine?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eYes that looks like it would be fine, what metrics are you tracking ? Do they all share those labels above?\u003c\/p\u003e\n\n\u003cp\u003eI'm asking because it some cases it might even make sense to move one of the labels with low cardinality into the metric name itself. e.g. if there are metrics that make sense only for anonymous or only for non-anonymous.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003e\u003cstrong\u003eMetric type\u003c\/strong\u003e\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003e\u003ca href=\"https:\/\/prometheus.io\/docs\/concepts\/metric_types\/#gauge\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/concepts\/metric_types\/\u003c\/a\u003e\u003cbr \/\u003e\nGauge\u003cbr \/\u003e\n\u003ca href=\"https:\/\/prometheus.io\/docs\/practices\/histograms\/\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/practices\/histograms\/\u003c\/a\u003e\u003cbr \/\u003e\nHistograms and summaries\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eIt sounds like Summaries is similar to what we do now \u2013 calculate the percentiles per-minute in StatsD (before sending to Graphite) at which points it becomes a single aggregated value, not appropiate for further aggregation.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCorrect, Summaries are like what we're currently doing with statsd\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eHistograms seem attractive given the query flexibility after the fact. Although the warning "not aggregatable" only applying to Summaries on that page, it sounds to me like it should apply to Histogram as well, given that it will not accurately tell you the median over 3 days in a way that considers all original values. It can accurately say in which bucket the value is (which is an improvement over our current Statsd\/Graphite approach), but it not being able to produce an actual number makes it hard to use for our use case, which is to catch small regressions in individual features and products, not just whether we're still meeting an SLA or not.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eI believe the "non aggregatable" in general refers to the fact that the aggregated results are still statistically meaningful. Note that aggregation here means if the same metric is pulled from more than one "target" (as prometheus calls it, a machine, a service, etc) to be then further processed by aggregation functions in prometheus. In the navtiming case ATM there's only one target from which the metrics are pulled, namely the machine where all measurements are funneled into. Meaning you could also use summaries, but it would break down in the case of multiple machines processing navtiming data.\u003c\/p\u003e\n\n\u003cp\u003eWRT to catching small regressions, you can "zoom in" the histogram buckets close to the latency target you're interested in and have additional precision that way. In particular I'm referring to something like the example case discussed here: \u003ca href=\"https:\/\/prometheus.io\/docs\/practices\/histograms\/#errors-of-quantile-estimation\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/prometheus.io\/docs\/practices\/histograms\/#errors-of-quantile-estimation\u003c\/a\u003e.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eGiven Prometheus' approach to not lose original input data, I'm leaning towards using a Gauge. Assuming the "we don't aggregate" principle applies there, that would mean all original data points are preserved and could be queried\/aggregated upon request. Although naturally I would imagine cannot be very good for query performance. Right?\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThat's correct, in general Prometheus isn't designed to store all observations (e.g. like a billing system) and query for exact results. Unfortunately the terminology gets confusing fast here (in the navtiming case the "input data" are also "datapoints", etc), but yeah it isn't feasible to store each individual navtiming observation sent by clients as distinct datapoints in Prometheus. Derived metrics from all input data are fine to store of course like discussed above.\u003c\/p\u003e\n\n\u003cp\u003eYou could do both though, i.e. have aggregated\/derived metrics from the input data as Prometheus metrics and store all received observations e.g. in hdfs to be queried via hive when exact results are needed.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"3777694\" id=\"3777694\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_114\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3777694\" data-sigil=\"has-tooltip\" data-meta=\"0_113\"\u003e\u003cspan class=\"screen-only\"\u003eNov 21 2017, 12:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2017-11-21 12:50:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_111\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_112\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T175087#3777688\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT175087#3777688\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@fgiunchedi\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eYou could do both though, i.e. have aggregated\/derived metrics from the input data as Prometheus metrics and store all received observations e.g. in hdfs to be queried via hive when exact results are needed.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eAs another example of possible pipeline involving storing and querying all input data, we're looking at storing netflow data in kafka and then into druid\/hdfs for querying (\u003ca href=\"\/T181036\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT181036\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_118\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3890057\" id=\"3890057\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_117\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003efgiunchedi\u003c\/a\u003e mentioned this in \u003ca href=\"\/T166173\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003eT166173: Upgrade graphite from 0.9.x to 1.x\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3890057\" data-sigil=\"has-tooltip\" data-meta=\"0_116\"\u003e\u003cspan class=\"screen-only\"\u003eJan 10 2018, 3:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-01-10 15:52:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_122\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/rus3onc75davarls6nbc\/PHID-FILE-d4tltleyv665iw455lvp\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Imarlier\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"3910228\" id=\"3910228\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Imarlier\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Imarlier\u003c\/a\u003e added a parent task: \u003ca href=\"\/T180105\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eT180105: Set up a statsv-like endpoint for Prometheus\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#3910228\" data-sigil=\"has-tooltip\" data-meta=\"0_119\"\u003e\u003cspan class=\"screen-only\"\u003eJan 18 2018, 5:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-01-18 17:20:38 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_121\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Imarlier\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e Imarlier\u003c\/a\u003e mentioned this in \u003ca href=\"\/T180105\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eT180105: Set up a statsv-like endpoint for Prometheus\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4142696\" id=\"4142696\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_130\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003egerritbot\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4142696\" data-sigil=\"has-tooltip\" data-meta=\"0_129\"\u003e\u003cspan class=\"screen-only\"\u003eApr 19 2018, 1:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-04-19 13:53:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_127\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_128\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_40\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 427664 had a related patch set uploaded (by Imarlier; owner: Imarlier):\u003cbr \/\u003e\n[performance\/coal@master] coal: convert to using graphite instead of writing to whisper\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/427664\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/427664\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_134\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4142697\" id=\"4142697\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_133\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003egerritbot\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/patch-for-review\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003ePatch-For-Review\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4142697\" data-sigil=\"has-tooltip\" data-meta=\"0_132\"\u003e\u003cspan class=\"screen-only\"\u003eApr 19 2018, 1:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-04-19 13:53:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_143\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"4146547\" id=\"4146547\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4146547\" data-sigil=\"has-tooltip\" data-meta=\"0_141\"\u003e\u003cspan class=\"screen-only\"\u003eApr 20 2018, 5:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-04-20 17:40:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_139\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 427664 \u003cstrong\u003emerged\u003c\/strong\u003e by jenkins-bot:\u003cbr \/\u003e\n[performance\/coal@master] coal: convert to using graphite instead of writing to whisper\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/427664\" class=\"remarkup-link remarkup-link-ext\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/427664\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_146\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/q4xtskw4ul5dvrupkmqs\/PHID-FILE-ezxrezgeehrb4vjobxgz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krinkle\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"4150551\" id=\"4150551\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_145\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krinkle\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eKrinkle\u003c\/a\u003e removed a parent task: \u003ca href=\"\/T159354\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003eT159354: Move coal from graphite#001 nodes to webperf#001\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#4150551\" data-sigil=\"has-tooltip\" data-meta=\"0_144\"\u003e\u003cspan class=\"screen-only\"\u003eApr 23 2018, 4:40 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2018-04-23 16:40:25 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz","contextPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz","contextPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yh7k5mn7fae4jk25jxiu"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr","contextPHID":"PHID-TASK-3lalowvshdibxsfuw77n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-h4bmfokzyzilbkwmpoxw"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-c23xzuwwbwxepcsmcv34"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-cu77k33bv4jo37k"},{"hovercardSpec":{"objectPHID":"PHID-USER-hfqxkl5wvmdkecl5utfy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-papbtlagfolot4dzerne"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-ylarhsvhxcxqbur6pbmv"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-mzxlpm2u56pgpkqn3onq"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-lci7ze4lrwahssnfkt7u"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jir5w4rxc52pxe7sggt4"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hfqxkl5wvmdkecl5utfy"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-c23xzuwwbwxepcsmcv34"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-c23xzuwwbwxepcsmcv34"}},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"phid":"PHID-XACT-TASK-dizxuri74uncko6"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-r6uv2uu23iatte6"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-6yoze5j2ug2k4h7"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-3j6ie3xby6y4rcnrcmrf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jzttt2rzt4bpbzkerkod"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-um4wlk7y6p5lfvfzs2mc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-jzttt2rzt4bpbzkerkod"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-um4wlk7y6p5lfvfzs2mc"}},{"phid":"PHID-XACT-TASK-ryhmjmvl5mum7fz"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-onnxucoedheq3jevknyr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-dnxx477eeditwpj"},{"hovercardSpec":{"objectPHID":"PHID-USER-sai77mtxmpqnm6pycyvz"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jir5w4rxc52pxe7sggt4"}},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eeugh6jn2ddbpkj","anchor":"3582425"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cwcxwl3vw4tvcrp","anchor":"3582515"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ysylweiircustm5","anchor":"3582604"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-4idqi4twtswihqi","anchor":"3583322"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cu77k33bv4jo37k\/","ref":"T175087#3583327"},[],{"anchor":"3583327"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_61\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_62\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cu77k33bv4jo37k\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_63\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_64\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-cu77k33bv4jo37k","anchor":"3583327"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ageui72rheis4wt","anchor":"3584744"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-x4zvsauc4aazpkg","anchor":"3599711"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rmew7zwoodnpltu","anchor":"3599742"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-hnn53k47lgnrodz","anchor":"3640903"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-wpl25lgka5fsrnq","anchor":"3775948"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dizxuri74uncko6\/","ref":"T175087#3776310"},[],{"anchor":"3776310"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_87\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_88\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dizxuri74uncko6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_89\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_90\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-dizxuri74uncko6","anchor":"3776310"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-r6uv2uu23iatte6\/","ref":"T175087#3777688"},[],{"anchor":"3777688"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_98\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_99\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-r6uv2uu23iatte6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_100\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_101\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-r6uv2uu23iatte6","anchor":"3777688"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6yoze5j2ug2k4h7\/","ref":"T175087#3777694"},[],{"anchor":"3777694"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_107\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_108\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6yoze5j2ug2k4h7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_109\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_110\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6yoze5j2ug2k4h7","anchor":"3777694"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mhthrpeuongqpqq","anchor":"3890057"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-7inwnnd5xkeymzb","anchor":"3910228"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ryhmjmvl5mum7fz\/","ref":"T175087#4142696"},[],{"anchor":"4142696"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_123\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_124\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ryhmjmvl5mum7fz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_125\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_126\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ryhmjmvl5mum7fz","anchor":"4142696"},{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-2nhca2jbmrvu3be","anchor":"4142697"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dnxx477eeditwpj\/","ref":"T175087#4146547"},[],{"anchor":"4146547"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_135\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_136\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dnxx477eeditwpj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-dnxx477eeditwpj","anchor":"4146547"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-abgfov2dbiygsxu","anchor":"4150551"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/dad1a225\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/286467bd\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/aa49028c\/rsrc\/css\/phui\/phui-badge.css"]}