for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2825088\" id=\"2825088\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-pencil phui-timeline-icon\" data-meta=\"0_94\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003efgiunchedi\u003c\/a\u003e created this task.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825088\" data-sigil=\"has-tooltip\" data-meta=\"0_93\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 4:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 04:48:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_98\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"display: none;\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2825098\" id=\"2825098\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_97\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003cspan class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-lock lightgreytext\" data-meta=\"0_22\" aria-hidden=\"true\"\u003e\u003c\/span\u003eRestricted Application\u003c\/span\u003e added a subscriber: \u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_23\"\u003eAklapper\u003c\/a\u003e. \u003cspan class=\"phui-timeline-extra-information\"\u003e \u00b7 \u003ca href=\"\/herald\/transcript\/1600277\/\"\u003eView Herald Transcript\u003c\/a\u003e\u003c\/span\u003e\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825098\" data-sigil=\"has-tooltip\" data-meta=\"0_96\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 4:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 04:48:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2825118\" id=\"2825118\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_106\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825118\" data-sigil=\"has-tooltip\" data-meta=\"0_105\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 6:13 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 06:13:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_103\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_104\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eLatest updates, tracked down to parsoid + euwiki requests by locally hacking mw1290 by ori. If those requests are excluded mem usage remains flat.\u003c\/p\u003e\n\n\u003cp\u003eAlso from a capture traffic (e.g. in P4516) all requests seem to have the same reqid \u003ctt class=\"remarkup-monospaced\"\u003ec7cbf760-07a5-41a2-90fa-1b7ed60fbb86\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_116\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2825119\" id=\"2825119\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_115\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825119\" data-sigil=\"has-tooltip\" data-meta=\"0_114\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 6:18 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 06:18:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_112\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_113\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003ePlugging that request id into logstash yields messages from parsoid, restbase and mobileapps, \u003ca href=\"https:\/\/logstash.wikimedia.org\/goto\/ceb222ade9b300c70a511686c23c41bd\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/logstash.wikimedia.org\/goto\/ceb222ade9b300c70a511686c23c41bd\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2825120\" id=\"2825120\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_124\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003efgiunchedi\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_29\"\u003eori\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825120\" data-sigil=\"has-tooltip\" data-meta=\"0_123\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 6:18 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 06:18:56 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_121\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_122\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_30\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_2\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e has applied a bandaid (\u003ca href=\"https:\/\/dpaste.de\/Py4m\/raw\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/dpaste.de\/Py4m\/raw\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_136\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_134\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_135\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2825123\" id=\"2825123\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_133\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003ebd808\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825123\" data-sigil=\"has-tooltip\" data-meta=\"0_132\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 6:31 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 06:31:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_130\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_131\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOne theory of what kicked this off:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ca href=\"https:\/\/eu.wikipedia.org\/wiki\/Modulu:Wikidata\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/eu.wikipedia.org\/wiki\/Modulu:Wikidata\u003c\/a\u003e was edited at 2016-11-26T22:42Z and again at 2016-11-26T23:20.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe \u003ca href=\"https:\/\/eu.wikipedia.org\/wiki\/Berezi:ZerkLotzenDuHona\/Modulu:Wikidata\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003e"what links here"\u003c\/a\u003e for that lua module is many thousands of pages long.\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eAs \u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_3\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e pointed out on irc edits of highly transcluded pages is pretty normal so there must be another contributing factor.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_147\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tz7lapf6xjud6l4n3hnu\/PHID-FILE-mc3f2ds2bpvzfeu4ogha\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/bd808\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_145\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_146\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2825124\" id=\"2825124\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_144\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/bd808\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_33\"\u003ebd808\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825124\" data-sigil=\"has-tooltip\" data-meta=\"0_143\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 6:37 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 06:37:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_141\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_142\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_34\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnother logstash view: \u003ca href=\"https:\/\/logstash.wikimedia.org\/goto\/2f822b03292d1350e2a98468ea54fd53\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/logstash.wikimedia.org\/goto\/2f822b03292d1350e2a98468ea54fd53\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_150\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c5mll4i2ivmjlggdpizk\/PHID-FILE-x2xqxc36bpfpfub7m6rd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Aklapper\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2825451\" id=\"2825451\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_149\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Aklapper\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_35\"\u003eAklapper\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/sre\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eSRE\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2825451\" data-sigil=\"has-tooltip\" data-meta=\"0_148\"\u003e\u003cspan class=\"screen-only\"\u003eNov 27 2016, 1:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-27 13:39:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell phui-timeline-green\" data-sigil=\"transaction anchor-container\" data-meta=\"0_154\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/ucqgevz7xnctw7dxwntl\/PHID-FILE-qwqnsn6e5ux4kqsbwulb\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Volans\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2826504\" id=\"2826504\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill fill-has-color phui-timeline-icon-fill-green\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-arrow-right phui-timeline-icon\" data-meta=\"0_152\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_37\"\u003eVolans\u003c\/a\u003e triaged this task as \u003cspan class=\"phui-timeline-value\"\u003eHigh\u003c\/span\u003e priority.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2826504\" data-sigil=\"has-tooltip\" data-meta=\"0_151\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 9:53 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 09:53:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_153\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Volans\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eVolans\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_39\"\u003eJoe\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_163\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2827422\" id=\"2827422\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_162\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_41\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2827422\" data-sigil=\"has-tooltip\" data-meta=\"0_161\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 4:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 16:27:17 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_159\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_160\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_40\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSome notes:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eIt seems that a very large percentage of Parsoid batch requests involving Modulu:Wikidata to HHVM \/ Scribunto timed out in Parsoid.\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe HHVM \/ Scribunto timeout seems to be larger than Parsoid's. We should consider lowering it to a value that is less than the client request, perhaps via \u003ca href=\"\/T97192\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_4\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT97192\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e. See \u003ca href=\"\/T97204\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT97204\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e for a general discussion of timeout staggering & retries.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe use of batch queries between Parsoid & the API might make timeouts more likely. If the entire batch is retried, work is duplicated.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eBased on looking at a few titles in the logstash dashboard, it seems that retry limits worked as designed, with at most three requests spaced out by at least six minutes for a given title from ChangeProp.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe duplicate request id used for different requests is odd. We should check where this is assigned. Using separate request ids for logically separate requests seems more appropriate for tasks like tracking retries.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_166\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/tnss5j5cboisfwvfyynt\/PHID-FILE-q4rxl5w7ormrj4e4lbcg\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/elukey\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2827572\" id=\"2827572\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_165\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/elukey\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eelukey\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2827572\" data-sigil=\"has-tooltip\" data-meta=\"0_164\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 5:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 17:05:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_177\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/5bz6h3qq7buznjhaa2nz\/PHID-FILE-ypguvfobi3lno5lpzjka\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/ori\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-indigo \" href=\"\/badges\/view\/6\/\" data-sigil=\"has-tooltip\" data-meta=\"0_175\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rebel\" data-meta=\"0_176\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2827618\" id=\"2827618\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_174\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/ori\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_43\"\u003eori\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2827618\" data-sigil=\"has-tooltip\" data-meta=\"0_173\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 5:21 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 17:21:47 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_171\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_172\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_44\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe expensive template appears to be \u003ca href=\"https:\/\/eu.wikipedia.org\/wiki\/Txantiloi:Frantziako_udalerri_infotaula_INSEE\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eTxantiloi:Frantziako_udalerri_infotaula_INSEE\u003c\/a\u003e. The titles on euwiki that Parsoid was requesting were all French municipalities. \u003ca href=\"https:\/\/eu.wikipedia.org\/wiki\/Guerchy\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003eGuerchy\u003c\/a\u003e, for example, takes almost a minute to parse, and reports these figures in its transclusion expansion time report:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eTransclusion expansion time report (%,ms,calls,template)\n100.00% 58495.609 1 - -total\n 98.66% 57713.565 1 - Txantiloi:Frantziako_udalerri_infotaula_INSEE\n 98.65% 57705.413 1 - Txantiloi:Infobox\n 96.44% 56413.554 102 - Txantiloi:Infotaula\/ilara\n 62.77% 36716.661 6 - Txantiloi:Data_bars\/bar\n 20.25% 11846.185 4 - Txantiloi:INSEE_errolda\/Kantonamendua\n 16.16% 9450.088 3 - Txantiloi:INSEE_errolda\/Kantonamendu_lotura\n 10.47% 6125.146 2 - Txantiloi:INSEE_errolda\/60-74\n 10.45% 6111.403 2 - Txantiloi:INSEE_errolda\/45-59\n 10.44% 6106.534 2 - Txantiloi:INSEE_errolda\/30-44\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo, that needs to be fixed.\u003c\/p\u003e\n\n\u003cp\u003eAs far as client behavior goes:\u003c\/p\u003e\n\n\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T151702#2827420\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT151702#2827420\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe duplicate request id used for different requests is odd. We should check where this is assigned. Using separate request ids for logically separate requests seems more appropriate for tasks like tracking retries.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWell, they clearly \u003cem\u003eare\u003c\/em\u003e logically related. They are all sub-requests made in the course of servicing one request \/ event in ChangeProp, and the person who wrote the code must have deemed it useful to have them all reference a shared identifier, precisely so they could be grouped.\u003c\/p\u003e\n\n\u003cp\u003eIdeally, the requests would each have a unique request ID, and reference the parent request in some other header or field.\u003c\/p\u003e\n\n\u003cp\u003eServices that handle requests by making many sub-requests should have a failure counter that is global to the parent request context, and they should abort if some threshold is crossed.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/2nco5z2tyvhf6dfxheoj\/PHID-FILE-pacxilbspughn725akdz\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/mark\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2827855\" id=\"2827855\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_179\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/mark\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_45\"\u003emark\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/tstarling\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003etstarling\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2827855\" data-sigil=\"has-tooltip\" data-meta=\"0_178\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 6:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 18:12:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_189\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828183\" id=\"2828183\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_188\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_47\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828183\" data-sigil=\"has-tooltip\" data-meta=\"0_187\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 7:04 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 19:04:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_185\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_186\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_48\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T151702#2827618\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT151702#2827618\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/ori\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@ori\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T151702#2827420\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT151702#2827420\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eThe duplicate request id used for different requests is odd. We should check where this is assigned. Using separate request ids for logically separate requests seems more appropriate for tasks like tracking retries.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWell, they clearly \u003cem\u003eare\u003c\/em\u003e logically related. They are all sub-requests made in the course of servicing one request \/ event in ChangeProp, and the person who wrote the code must have deemed it useful to have them all reference a shared identifier, precisely so they could be grouped.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eRequest ids are normally intended to trace a single, synchronous request through the infrastructure. Using a single request id for potentially millions of derived updates as well makes it difficult to distinguish primary request processing from derived updates.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eIdeally, the requests would each have a unique request ID, and reference the parent request in some other header or field.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThis is the idea behind the \u003ctt class=\"remarkup-monospaced\"\u003etriggered-by\u003c\/tt\u003e header added in ChangeProp, which contains a reference to the original event that triggered asynchronous updates. IIRC it contains a URL, so can also include request ids. By separating the two levels, it becomes possible to clearly distinguish primary event from derived event processing.\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eServices that handle requests by making many sub-requests should have a failure counter that is global to the parent request context, and they should abort if some threshold is crossed.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eCould you elaborate a bit on what you mean with "abort"? How would this work when other temporary infrastructure issues cause bursts of failures?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828577\" id=\"2828577\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_197\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_49\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828577\" data-sigil=\"has-tooltip\" data-meta=\"0_196\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:27:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_194\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_195\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_50\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWe've discussed this too a bit on today's ops meeting, to summarize:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list remarkup-list-with-checkmarks\"\u003e\n\u003cli class=\"remarkup-list-item remarkup-unchecked-item\"\u003e\u003cinput type=\"checkbox\" disabled=\"disabled\" \/\u003e I'll start on an incident report (and \u003ca href=\"\/T148652\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT148652: 2016-10-17 API cluster overload\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e which seems like a similar reoccurence back in Oct)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item remarkup-unchecked-item\"\u003e\u003cinput type=\"checkbox\" disabled=\"disabled\" \/\u003e The API cluster needs better isolation, the simplest way for now is to split the cluster by usage (e.g. internal vs external clients) to reduce an outage blast radius.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item remarkup-unchecked-item\"\u003e\u003cinput type=\"checkbox\" disabled=\"disabled\" \/\u003e On the HHVM side, issues like this would benefit from memory profiling. AFAIK it is disabled because it proved to be problematic in the past, we'd need to reevaluate and\/or figure next steps out with upstream.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item remarkup-unchecked-item\"\u003e\u003cinput type=\"checkbox\" disabled=\"disabled\" \/\u003e On the MW side we'd need to figure out if there's anything we can do to mitigate similar problems in templates\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item remarkup-unchecked-item\"\u003e\u003cinput type=\"checkbox\" disabled=\"disabled\" \/\u003e On the clients\/API side various timeout\/retries tasks and request-id work that \u003ca href=\"\/p\/GWicke\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e\u003cspan class=\"phui-tag-dot phui-tag-color-grey\"\u003e\u003c\/span\u003e@GWicke\u003c\/span\u003e\u003c\/a\u003e linked earlier\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_207\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828587\" id=\"2828587\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_206\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_51\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828587\" data-sigil=\"has-tooltip\" data-meta=\"0_205\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:29 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:29:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_203\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_204\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_52\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eBacktraces from mw1279 at the time of the outage in \u003ca href=\"\/P4528\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4528\u003c\/span\u003e\u003c\/a\u003e and \u003ca href=\"\/P4529\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4529\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_210\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/p2jvmcdsbef3436hkcf6\/PHID-FILE-b6vmtimun4dtm56lypjx\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Krenair\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2828601\" id=\"2828601\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_209\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Krenair\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_53\"\u003eKrenair\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/wmf-nda\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003eWMF-NDA\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828601\" data-sigil=\"has-tooltip\" data-meta=\"0_208\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:30:48 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_213\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2828602\" id=\"2828602\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_212\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_55\"\u003efgiunchedi\u003c\/a\u003e added projects: \u003ca href=\"\/tag\/release-engineering-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003eRelease-Engineering-Team\u003c\/a\u003e, \u003ca href=\"\/tag\/hhvm\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_57\"\u003eHHVM\u003c\/a\u003e, \u003ca href=\"\/tag\/wikimedia-incident\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eWikimedia-Incident\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828602\" data-sigil=\"has-tooltip\" data-meta=\"0_211\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:30:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_223\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828607\" id=\"2828607\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_222\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_59\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2828607\" data-sigil=\"has-tooltip\" data-meta=\"0_221\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:35 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:35:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_219\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_60\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYet more backtraces, \u003ca href=\"\/P4530\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4530\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/P4531\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4531\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/P4532\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4532\u003c\/span\u003e\u003c\/a\u003e \u003ca href=\"\/P4533\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4533\u003c\/span\u003e\u003c\/a\u003e (not from mw1279)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_232\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828755\" id=\"2828755\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_61\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828755\" data-sigil=\"has-tooltip\" data-meta=\"0_230\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:53:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_228\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_62\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eReporting the patch here too to avoid relying on external paste\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"diff\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gh\"\u003ediff --git a\/api.php b\/api.php\u003c\/span\u003e\n\u003cspan class=\"gh\"\u003eindex 6e75fb7..f40e235 100644\u003c\/span\u003e\n\u003cspan class=\"gd\"\u003e--- a\/api.php\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+++ b\/api.php\u003c\/span\u003e\n\u003cspan class=\"gu\"\u003e@@ -32,6 +32,15 @@\u003c\/span\u003e\n \n use MediaWiki\\Logger\\LegacyLogger;\n \n\u003cspan class=\"gi\"\u003e+if (\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+ ( isset( $_SERVER['HTTP_USER_AGENT'] ) && strpos( $_SERVER['HTTP_USER_AGENT'], 'Parsoid' ) !== false ) &&\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+ ( isset( $_SERVER['HTTP_HOST'] ) && $_SERVER['HTTP_HOST'] === 'eu.wikipedia.org' )\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+) {\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+ header( $_SERVER['SERVER_PROTOCOL'] . ' 500 T151702 Bandaid', true, 500 );\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+ die( 1 );\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+}\u003c\/span\u003e\n\u003cspan class=\"gi\"\u003e+\u003c\/span\u003e\n \/\/ So extensions (and other code) can check whether they're running in API mode\n define( 'MW_API', true );\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_241\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828786\" id=\"2828786\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_63\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828786\" data-sigil=\"has-tooltip\" data-meta=\"0_239\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 8:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 20:59:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_237\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_238\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_64\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've now reverted the above bandaid in production\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_254\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/nnuo46yu5fgx4an24z46\/PHID-FILE-mupbjx7vjqa5hurxlggi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/hashar\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-blue \" href=\"\/badges\/view\/11\/\" data-sigil=\"has-tooltip\" data-meta=\"0_250\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-space-shuttle\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/8\/\" data-sigil=\"has-tooltip\" data-meta=\"0_252\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-life-ring\" data-meta=\"0_253\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2828944\" id=\"2828944\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/hashar\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003ehashar\u003c\/a\u003e subscribed.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2828944\" data-sigil=\"has-tooltip\" data-meta=\"0_248\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 9:26 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 21:26:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_246\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOri wrote:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eThe expensive template appears to be Txantiloi:Frantziako_udalerri_infotaula_INSEE. The titles on euwiki that Parsoid was requesting were all French municipalities. [..]\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eA bit of context\/history:\u003c\/p\u003e\n\n\u003cp\u003eINSEE is the France state agency in charges of statistics. In 2012, the french wiki wanted to import population history of all 36000 cities from France. The idea was to get several layers of templates to hold the data and a metatemplate on top of it to look them up. They do million of \u003ctt class=\"remarkup-monospaced\"\u003e#switch\u003c\/tt\u003e calls and load the whole data set in memory! You can get an idea of an underlying "data store" at \u003ca href=\"https:\/\/eu.wikipedia.org\/w\/index.php?title=Txantiloi:INSEE_errolda\/45-59&action=edit&uselang=en\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/eu.wikipedia.org\/w\/index.php?title=Txantiloi:INSEE_errolda\/45-59&action=edit&uselang=en\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eTim wrote a summary at \u003ca href=\"https:\/\/lists.wikimedia.org\/pipermail\/wikitech-l\/2012-September\/063360.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/lists.wikimedia.org\/pipermail\/wikitech-l\/2012-September\/063360.html\u003c\/a\u003e and I was tasked in relaying the message to the french wikipedia community (\u003ca href=\"https:\/\/fr.wikipedia.org\/wiki\/Wikip%C3%A9dia:Le_Bistro\/20_septembre_2012#Donn.C3.A9es_g.C3.A9ographiques_et_charges_sur_les_serveurs\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003emy message on french village pupmp, Sept 2012\u003c\/a\u003e).\u003c\/p\u003e\n\n\u003cp\u003eTim wrote:\u003c\/p\u003e\n\n\u003cblockquote\u003e\u003cp\u003eHere is an example of a problematic template:\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/fr.wikipedia.org\/w\/index.php?title=Mod%C3%A8le:Donn%C3%A9es_PyrF1-2009&action=edit\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/fr.wikipedia.org\/w\/index.php?title=Mod%C3%A8le:Donn%C3%A9es_PyrF1-2009&action=edit\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eThat template alone uses 47MB for 37000 #switch cases, and one article used about 15 similarly sized templates.\u003c\/p\u003e\u003c\/blockquote\u003e\n\n\u003cp\u003eThe template on the euwiki is most definitely a copy paste from that era. I have no idea how the frwiki worked around it, maybe by using a LUA module, getting rid of the template, switching to wikidata or whatever else.\u003c\/p\u003e\n\n\u003cp\u003eIt is most probably worth filling a follow up task to identify how it has been solved on the french wiki and work on a similar solution on euwiki. Ideally look at whether a LUA module could fetch the census data from wikidata.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_258\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/k5q3d2m3ejyx7cobtbbl\/PHID-FILE-4rln7zwcyy6tvhyo2na3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/GWicke\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2829049\" id=\"2829049\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_67\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a project: \u003ca href=\"\/tag\/parsing-team--archived\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003eParsing-Team--ARCHIVED\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829049\" data-sigil=\"has-tooltip\" data-meta=\"0_255\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 9:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 21:47:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-user-plus phui-timeline-icon\" data-meta=\"0_257\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/GWicke\/\" class=\"phui-handle handle-availability-disabled phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_69\"\u003e\u003cspan class=\"perfect-circle\"\u003e\u2022\u003c\/span\u003e GWicke\u003c\/a\u003e added a subscriber: \u003ca href=\"\/p\/ssastry\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003essastry\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_267\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829319\" id=\"2829319\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_266\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_71\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829319\" data-sigil=\"has-tooltip\" data-meta=\"0_265\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 11:14 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 23:14:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_263\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_264\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_72\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIncident report draft for this and Oct's API outages, \u003ca href=\"https:\/\/wikitech.wikimedia.org\/wiki\/Incident_documentation\/20161127-API\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/wikitech.wikimedia.org\/wiki\/Incident_documentation\/20161127-API\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_271\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/sdb5e3nkynpf4swmwiza\/PHID-FILE-cvbkyayzlwbwsqgg7q4f\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/greg\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"2829404\" id=\"2829404\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/greg\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_73\"\u003egreg\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/2143\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eActive investigation\u003c\/a\u003e to \u003ca href=\"\/project\/board\/2143\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_75\"\u003eActive Situation\u003c\/a\u003e on the \u003ca href=\"\/tag\/wikimedia-incident\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eWikimedia-Incident\u003c\/a\u003e board.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829404\" data-sigil=\"has-tooltip\" data-meta=\"0_268\"\u003e\u003cspan class=\"screen-only\"\u003eNov 28 2016, 11:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-28 23:51:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-columns phui-timeline-icon\" data-meta=\"0_270\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/greg\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_77\"\u003egreg\u003c\/a\u003e moved this task from \u003ca href=\"\/project\/board\/20\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003eINBOX\u003c\/a\u003e to \u003ca href=\"\/project\/board\/20\/\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_79\"\u003eExternally Blocked \/ Watching\u003c\/a\u003e on the \u003ca href=\"\/tag\/release-engineering-team\/\" class=\"phui-handle\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003eRelease-Engineering-Team\u003c\/a\u003e board.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_280\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/c6xmpug7aawzqcut5d3f\/PHID-FILE-v2zy4knvlloiceegtfy7\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/fgiunchedi\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829535\" id=\"2829535\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_279\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/fgiunchedi\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_81\"\u003efgiunchedi\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829535\" data-sigil=\"has-tooltip\" data-meta=\"0_278\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2016, 12:54 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-29 00:54:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_276\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_277\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_82\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMore backtraces, \u003ca href=\"\/P4535\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eP4535\u003c\/span\u003e\u003c\/a\u003e from mw1287\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_290\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xgl3wy55ovinfnucvwhx\/PHID-FILE-zocnksypiqutzf5oubpi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/tstarling\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829617\" id=\"2829617\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_289\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/tstarling\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_83\"\u003etstarling\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2829617\" data-sigil=\"has-tooltip\" data-meta=\"0_288\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2016, 2:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-29 02:22:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_286\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_84\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt's not clear from the backtraces what sort of lock is the subject of contention. They show JSON_parser() calling __lll_lock_wait(), but the return address indicates that it is actually in json_tokener_parse_ex, in the jsonc library, which is compiled with frame pointers omitted.\u003c\/p\u003e\n\n\u003cp\u003e\u003cdel\u003eOne possibility is the fact that json_tokener_parse_ex() calls setlocale() on entry and exit. This is obviously broken in a multithreaded program. This was fixed in the git master version of json-c \u003ca href=\"https:\/\/github.com\/json-c\/json-c\/commit\/4091b9c87ea293fd8bc3b05bc5b032e09697eefe\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ein July\u003c\/a\u003e, but there has not been a tarball release yet.\u003c\/del\u003e [UPDATE: ruled this out]\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_299\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xgl3wy55ovinfnucvwhx\/PHID-FILE-zocnksypiqutzf5oubpi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/tstarling\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829680\" id=\"2829680\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_298\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/tstarling\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_85\"\u003etstarling\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829680\" data-sigil=\"has-tooltip\" data-meta=\"0_297\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2016, 3:59 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-29 03:59:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_295\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_86\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe atop data on mw1279 does show some HHVM memory spikes, which are apparently not reflected in the Ganglia HHVM VmSize\/VmRss metrics. Not enough to be described as "out of memory" -- maybe 34GB maximum RSS out of 64GB physical (this was reached at 02:20, 02:40, 05:00, 05:20 and 06:20). But I guess it is enough to cause lock contention in the allocator. CPU usage was 40-50%, which is higher than normal.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_308\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xgl3wy55ovinfnucvwhx\/PHID-FILE-zocnksypiqutzf5oubpi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/tstarling\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829691\" id=\"2829691\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_307\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/tstarling\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_87\"\u003etstarling\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829691\" data-sigil=\"has-tooltip\" data-meta=\"0_306\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2016, 4:44 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-29 04:44:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_304\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_88\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eOn the community side, it looks like An13sa\/An13saBot is primarily responsible:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ca href=\"https:\/\/eu.wikipedia.org\/w\/index.php?title=Paris&diff=5324817&oldid=5186402&uselang=en\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003emigrating Paris to INSEE by hand in February\u003c\/a\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003e\u003ca href=\"https:\/\/eu.wikipedia.org\/w\/index.php?title=Berezi:Ekarpenak&offset=20160520000000&limit=500&contribs=user&target=An13saBot&namespace=&tagfilter=\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003emigrating hundreds of towns to INSEE in May\u003c\/a\u003e\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eThis user has an en-3 userbox and so it may be possible to talk him\/her out of this course of action.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_318\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/xgl3wy55ovinfnucvwhx\/PHID-FILE-zocnksypiqutzf5oubpi\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/tstarling\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829704\" id=\"2829704\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/tstarling\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_89\"\u003etstarling\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#2829704\" data-sigil=\"has-tooltip\" data-meta=\"0_316\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2016, 5:40 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-29 05:40:22 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_314\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_315\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_90\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnd on the parser side: the template invocation in P4516 has a generated node count of about 2M, and so fails with an exception when expanded with Preprocessor_DOM which has a limit of 1.5M. This is the ocwiki hack, intended to prevent OOM. But that limit is not implemented for Preprocessor_Hash, due to the fact that there's no single place where nodes can be efficiently counted.\u003c\/p\u003e\n\n\u003cp\u003ePerhaps count( $rootStore, COUNT_RECURSIVE ) \/ 2 would be a reasonable estimate?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_329\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/4vfodng5xaao26r6l2bf\/PHID-FILE-2gvu7et6e6bs2cswr2hw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Joe\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-badges\"\u003e\u003cul class=\"phui-badge-flex-view grouped flex-view-collapsed \"\u003e\u003cli class=\"phui-badge-flex-item\"\u003e\u003ca class=\"phui-badge-mini phui-badge-mini-orange \" href=\"\/badges\/view\/17\/\" data-sigil=\"has-tooltip\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-rocket\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"2829927\" id=\"2829927\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Joe\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_91\"\u003eJoe\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#2829927\" data-sigil=\"has-tooltip\" data-meta=\"0_325\"\u003e\u003cspan class=\"screen-only\"\u003eNov 29 2016, 8:54 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2016-11-29 08:54:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_323\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_324\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_92\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eExpanding a bit on the concept of having a separate cluster:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhatever spawns from ChangeProp is an async job, not part of serving content to actual, live users\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eFor async processing, we have a separate cluster in mediawiki, and that's the jobrunners cluster. We're not doing the same with Changeprop.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eWhile rate-limiting on the Changeprop side is probably a good thing to tackle, getting every edge case right is probably very tricky.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eSeparation of concerns will never be fully ironclad (databases will still be shared after all), but it would at least confine events like the current one to async processing\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSo my personal view is as follows: for most systems, requests originated by changeprop should not be served by the same servers that are serving the traffic from live users.\u003c\/p\u003e\n\n\u003cp\u003eThis can be very tricky and depend on a lot of details about how every call is made, but let's focus on MediaWiki though: how do calls to MW originate indirectly from changeprop? how can we divert them to a different cluster?\u003c\/p\u003e\n\n\u003cp\u003eIn my understanding, what happens most commonly is:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eChangeProp asks Restbase to re-render some content\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eRestbase forwards the re-render request to Parsoid\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eParsoid forwards it to the mediawiki api\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eSo the easiest way to do this would be to separate a few servers (how many, to be determined) from the restbase, parsoid, and mediawiki api clusters and dedicate those to this specific task. I don't see any better solution, to be honest, unless we put a L7 proxy between the service layers, which IMHO would be a worse solution.\u003c\/p\u003e\n\n\u003cp\u003eI'll use the morning to assess which percentage of requests to the various clusters are originated by changeprop, to understand what the scope of the change might need to be.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-47waed4g56mtw7zc26rc"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-uoulnf5r3m2ek3uf73ex"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g","contextPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-jjqcfbi2zy3xjgptnrkm"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf","contextPHID":"PHID-TASK-zqe4j6gkntgjghyrfqp2"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-ahlouky24lslucdndxin"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-ny77hl63fqvqvtbixb4j"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-525r2f6ujdnl6zrw2oax"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-ei5t75zfgwt64vwfzkkd"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-ytfcg666t45h4djd6b3h"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-rdd4zt6v6hybr6ry4vlx"}},{"hovercardSpec":{"objectPHID":"PHID-PSTE-yoe3rl6u4ux5x3uegl6t"}},{"hovercardSpec":{"objectPHID":"PHID-APPS-PhabricatorHeraldApplication"}},[],{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-6bdgm7sxopwbttf"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-o3l675ducx7cef7"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-ju4ini3xoi7c62r"},{"phid":"PHID-XACT-TASK-3av6geqb3ks252j"},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"hovercardSpec":{"objectPHID":"PHID-USER-ll6tmaogat2b5q7tnqas"}},{"phid":"PHID-XACT-TASK-t2jmcxvklvyfhom"},{"hovercardSpec":{"objectPHID":"PHID-USER-hgn5uw2jafgjgfvxibhh"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-5hj6ygnanfu23mmnlvmd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-qbtoxfmd3v2vpn7foso2"}},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-gi2uzf53pzngcxn"},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-a6yycs7vaq2biua4pudd"}},{"hovercardSpec":{"objectPHID":"PHID-USER-r7wrkcx7j2vutqs6hr3g"}},{"phid":"PHID-XACT-TASK-rfdrrjcixmq5myx"},{"hovercardSpec":{"objectPHID":"PHID-USER-hovi6c77qompv5qd7gtz"}},{"hovercardSpec":{"objectPHID":"PHID-USER-k6tmz5ylx4rzfl3bitse"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"phid":"PHID-XACT-TASK-uj4cveemh425rva"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-eekcg2inhmziak6"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-o4kgktwdebm6usx"},{"hovercardSpec":{"objectPHID":"PHID-USER-x7ti5ksby4ubsabntlxa"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-ibxm3v6ithf3jpqpqhl7"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-uier7rukzszoewbhj7ja"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-6deovqxiehg3nlp7fdj4"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-jpysj5rksyrh25knsg4n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-rnabjnvopaf4bcr"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-vmhcitnz3htuwrn"},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-p4ebjl37dcgppao"},{"phid":"PHID-XACT-TASK-27m6xx6xzkhoqyd"},{"hovercardSpec":{"objectPHID":"PHID-USER-orzyp3dswemhdgdznro5"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-rzwhdsobhl4pf44og662"}},{"hovercardSpec":{"objectPHID":"PHID-USER-hbtlbu4zftxnz4i6f7yf"}},{"hovercardSpec":{"objectPHID":"PHID-USER-slccyo5rqasgpljxny7g"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-m436nvqpmuhuzzq"},{"hovercardSpec":{"objectPHID":"PHID-USER-z3l4i7dl52qicxtephy5"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-zz2tn24i3nfz6iuoz54k"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-hgow44esems3ezhtpdna"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-jpysj5rksyrh25knsg4n"}},{"hovercardSpec":{"objectPHID":"PHID-USER-z3l4i7dl52qicxtephy5"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-vcu3kr65w3n4bmi6nzdq"}},{"hovercardSpec":{"objectPHID":"PHID-PCOL-lhig7rhsiidogudnq45a"}},{"hovercardSpec":{"objectPHID":"PHID-PROJ-uier7rukzszoewbhj7ja"}},{"hovercardSpec":{"objectPHID":"PHID-USER-wr3pospifdaxghmtobvr"}},{"phid":"PHID-XACT-TASK-wcyhopdducptqjr"},{"hovercardSpec":{"objectPHID":"PHID-USER-k6tmz5ylx4rzfl3bitse"}},{"phid":"PHID-XACT-TASK-xptbhmapy25whft"},{"hovercardSpec":{"objectPHID":"PHID-USER-k6tmz5ylx4rzfl3bitse"}},{"phid":"PHID-XACT-TASK-emls5otd5udzavv"},{"hovercardSpec":{"objectPHID":"PHID-USER-k6tmz5ylx4rzfl3bitse"}},{"phid":"PHID-XACT-TASK-nyolpwxzkkz2f44"},{"hovercardSpec":{"objectPHID":"PHID-USER-k6tmz5ylx4rzfl3bitse"}},{"phid":"PHID-XACT-TASK-wgg3aneaoxlo65e"},{"hovercardSpec":{"objectPHID":"PHID-USER-fdo23otm6ztt674vjqko"}},{"phid":"PHID-XACT-TASK-ymnorkcb74ipm6c"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ke5xnwgqqwd5dr4","anchor":"2825088"},{"tip":"Via Herald"},[],{"phid":"PHID-XACT-TASK-cprioz75zr6uilj","anchor":"2825098"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-6bdgm7sxopwbttf\/","ref":"T151702#2825118"},[],{"anchor":"2825118"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_99\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_100\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-6bdgm7sxopwbttf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_101\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_102\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6bdgm7sxopwbttf","anchor":"2825118"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-o3l675ducx7cef7\/","ref":"T151702#2825119"},[],{"anchor":"2825119"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_108\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_109\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-o3l675ducx7cef7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_110\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_111\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-o3l675ducx7cef7","anchor":"2825119"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ju4ini3xoi7c62r\/","ref":"T151702#2825120"},[],{"anchor":"2825120"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_117\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_118\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ju4ini3xoi7c62r\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_119\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_120\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4m2gwvf7feinqpv","anchor":"2825120"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3av6geqb3ks252j\/","ref":"T151702#2825123"},[],{"anchor":"2825123"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_126\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_127\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3av6geqb3ks252j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_128\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_129\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-3av6geqb3ks252j","anchor":"2825123"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-t2jmcxvklvyfhom\/","ref":"T151702#2825124"},[],{"anchor":"2825124"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_137\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_138\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t2jmcxvklvyfhom\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_139\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_140\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-t2jmcxvklvyfhom","anchor":"2825124"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xnw6xqzmhhwm265","anchor":"2825451"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-qjeqhstp56ngflu","anchor":"2826504"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gi2uzf53pzngcxn\/","ref":"T151702#2827422"},[],{"anchor":"2827422"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_155\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_156\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gi2uzf53pzngcxn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_157\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_158\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gi2uzf53pzngcxn","anchor":"2827422"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-s6yax24t5dm2tzd","anchor":"2827572"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rfdrrjcixmq5myx\/","ref":"T151702#2827618"},[],{"anchor":"2827618"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_167\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_168\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rfdrrjcixmq5myx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_169\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_170\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Chaos Monkey","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-rfdrrjcixmq5myx","anchor":"2827618"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4nefdqeb6pgt54g","anchor":"2827855"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uj4cveemh425rva\/","ref":"T151702#2828183"},[],{"anchor":"2828183"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_181\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_182\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uj4cveemh425rva\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_183\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_184\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uj4cveemh425rva","anchor":"2828183"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-eekcg2inhmziak6\/","ref":"T151702#2828577"},[],{"anchor":"2828577"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_190\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_191\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eekcg2inhmziak6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_192\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_193\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-eekcg2inhmziak6","anchor":"2828577"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-o4kgktwdebm6usx\/","ref":"T151702#2828587"},[],{"anchor":"2828587"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_199\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_200\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-o4kgktwdebm6usx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_201\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_202\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-o4kgktwdebm6usx","anchor":"2828587"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mqr542cjvddbddx","anchor":"2828601"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-enp3n7gaisrtebc","anchor":"2828602"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rnabjnvopaf4bcr\/","ref":"T151702#2828607"},[],{"anchor":"2828607"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_214\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_215\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rnabjnvopaf4bcr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_216\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_217\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-rnabjnvopaf4bcr\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rnabjnvopaf4bcr","anchor":"2828607"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-vmhcitnz3htuwrn\/","ref":"T151702#2828755"},[],{"anchor":"2828755"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_224\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_225\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-vmhcitnz3htuwrn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-vmhcitnz3htuwrn","anchor":"2828755"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-p4ebjl37dcgppao\/","ref":"T151702#2828786"},[],{"anchor":"2828786"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_233\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_234\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-p4ebjl37dcgppao\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_235\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-p4ebjl37dcgppao","anchor":"2828786"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-27m6xx6xzkhoqyd\/","ref":"T151702#2828944"},[],{"anchor":"2828944"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_242\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_243\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-27m6xx6xzkhoqyd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_244\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Train Conductor","align":"E","size":300},[],{"tip":"Continuous Integrator","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-27m6xx6xzkhoqyd","anchor":"2828944"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-glhf66cu4wgoq6z","anchor":"2829049"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-m436nvqpmuhuzzq\/","ref":"T151702#2829319"},[],{"anchor":"2829319"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_259\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-m436nvqpmuhuzzq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_261\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_262\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-m436nvqpmuhuzzq","anchor":"2829319"},{"tip":"Via Web"},[],[],{"phid":"PHID-XACT-TASK-t5bufingjs6onpo","anchor":"2829404"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wcyhopdducptqjr\/","ref":"T151702#2829535"},[],{"anchor":"2829535"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_272\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_273\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wcyhopdducptqjr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_274\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_275\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wcyhopdducptqjr","anchor":"2829535"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xptbhmapy25whft\/","ref":"T151702#2829617"},[],{"anchor":"2829617"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_281\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_282\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xptbhmapy25whft\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_283\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_284\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-xptbhmapy25whft\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xptbhmapy25whft","anchor":"2829617"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-emls5otd5udzavv\/","ref":"T151702#2829680"},[],{"anchor":"2829680"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_291\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_292\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-emls5otd5udzavv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_293\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-emls5otd5udzavv","anchor":"2829680"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nyolpwxzkkz2f44\/","ref":"T151702#2829691"},[],{"anchor":"2829691"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_300\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nyolpwxzkkz2f44\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_302\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nyolpwxzkkz2f44","anchor":"2829691"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-wgg3aneaoxlo65e\/","ref":"T151702#2829704"},[],{"anchor":"2829704"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_309\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-wgg3aneaoxlo65e\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_311\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-wgg3aneaoxlo65e\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_313\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-wgg3aneaoxlo65e","anchor":"2829704"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ymnorkcb74ipm6c\/","ref":"T151702#2829927"},[],{"anchor":"2829927"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_319\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_320\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ymnorkcb74ipm6c\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_321\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_322\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"tip":"Web Perf Hero","align":"E","size":300},[],{"phid":"PHID-XACT-TASK-ymnorkcb74ipm6c","anchor":"2829927"}],"javelin_behaviors":{"phui-hovercards":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[],"phui-dropdown-menu":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/666e25ad\/rsrc\/css\/phui\/phui-badge.css"]}