for (;;);{"error":null,"payload":{"timeline":"\u003cdiv class=\"phui-timeline-older-transactions-are-hidden\" data-sigil=\"show-older-block\"\u003eThere are a very large number of changes, so older changes are hidden. \u003ca href=\"\/transactions\/showolder\/PHID-TASK-vzliwhqljwra2fkn5a76\/?after=5301723&quoteTargetID=UQ0_1&quoteRef=T215531\" data-mustcapture=\"1\" data-sigil=\"show-older-link\"\u003eShow Older Changes\u003c\/a\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_225\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301723\" id=\"5301723\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_224\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_0\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301723\" data-sigil=\"has-tooltip\" data-meta=\"0_223\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 7:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 19:54:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_221\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_222\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_23\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520265 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: add a join configuration to the init setup\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520265\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520265\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_234\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301739\" id=\"5301739\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_233\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_24\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301739\" data-sigil=\"has-tooltip\" data-meta=\"0_232\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 7:58 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 19:58:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_230\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_231\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_25\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520303 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: reload haproxy on subconfig changes\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520303\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520303\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_243\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301811\" id=\"5301811\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_242\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_26\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301811\" data-sigil=\"has-tooltip\" data-meta=\"0_241\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 8:03 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 20:03:52 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_239\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_240\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_27\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520306 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: a firewall default of DENY is maddening for kubernetes\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520306\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520306\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_252\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301901\" id=\"5301901\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_251\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_28\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301901\" data-sigil=\"has-tooltip\" data-meta=\"0_250\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 8:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 20:39:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_248\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_249\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_29\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWithout the firewall, it's up!\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ echo "show stat" | socat \/run\/haproxy\/haproxy.sock stdio\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,agent_status,agent_code,agent_duration,check_desc,agent_desc,check_rise,check_fall,check_health,agent_rise,agent_fall,agent_health,addr,cookie,mode,algo,conn_rate,conn_rate_max,conn_tot,intercepted,dcon,dses,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,FRONTEND,,,1,24,2000,74005,9793063,27840096,0,0,0,,,,,OPEN,,,,,,,,,1,2,0,,,,0,0,0,23,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,,,,,,,,,,,,,,tcp,,0,23,74005,,0,0,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,toolsbeta-test-k8s-master-1,0,0,1,22,250,1876,2560643,27840096,,0,,341,0,1220,18,UP,100,1,0,13,7,93,5115,256,1,3,1,,637,,2,0,,25,L4OK,,1,,,,,,,,,,,84,0,,,,,17,,,112,234,0,6472,,,,Layer4 check passed,,2,2,3,,,,,,tcp,,,,,,,,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,BACKEND,0,12,1,23,200,74005,9793063,27840096,0,0,,73657,0,1220,18,UP,100,1,0,,7,93,5115,,1,3,0,,637,,1,0,,23,,,,,,,,,,,,,,84,0,0,0,0,0,17,,,112,234,0,6472,,,,,,,,,,,,,,tcp,,,,,,,,\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_261\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301932\" id=\"5301932\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_260\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_30\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301932\" data-sigil=\"has-tooltip\" data-meta=\"0_259\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 8:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 20:54:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_257\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_258\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_31\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt took it a little while to start behaving after all this, but:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# kubectl apply -f \/etc\/kubernetes\/calico.yaml \nconfigmap\/calico-config created\ncustomresourcedefinition.apiextensions.k8s.io\/felixconfigurations.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/ipamblocks.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/blockaffinities.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/ipamhandles.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/ipamconfigs.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/bgppeers.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/bgpconfigurations.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/ippools.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/hostendpoints.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/clusterinformations.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/globalnetworkpolicies.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/globalnetworksets.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/networkpolicies.crd.projectcalico.org created\ncustomresourcedefinition.apiextensions.k8s.io\/networksets.crd.projectcalico.org created\nclusterrole.rbac.authorization.k8s.io\/calico-kube-controllers created\nclusterrolebinding.rbac.authorization.k8s.io\/calico-kube-controllers created\nclusterrole.rbac.authorization.k8s.io\/calico-node created\nclusterrolebinding.rbac.authorization.k8s.io\/calico-node created\ndaemonset.apps\/calico-node created\nserviceaccount\/calico-node created\ndeployment.apps\/calico-kube-controllers created\nserviceaccount\/calico-kube-controllers created\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_270\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301942\" id=\"5301942\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_269\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_32\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301942\" data-sigil=\"has-tooltip\" data-meta=\"0_268\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:01:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_266\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_267\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_33\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat apply above went through the load balancer :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_279\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301987\" id=\"5301987\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_278\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_34\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301987\" data-sigil=\"has-tooltip\" data-meta=\"0_277\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:20 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:20:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_275\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_276\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_35\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520319 had a related patch set uploaded (by Bstorm; owner: Bstorm):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: refactor to join a node to the new cluster\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520319\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520319\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_288\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5301990\" id=\"5301990\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_287\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_36\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5301990\" data-sigil=\"has-tooltip\" data-meta=\"0_286\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:22 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:22:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_284\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_285\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_37\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eHmmm. It worked for a bit, but now it is strangely giving me EOFs instead of replies from the api.  Investigating that.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_297\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302016\" id=\"5302016\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_296\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_38\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302016\" data-sigil=\"has-tooltip\" data-meta=\"0_295\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:27:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_293\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_294\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_39\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAaand now it's working:\u003cbr \/\u003e\nroot@toolsbeta-test-k8s-master-1:~# kubectl get nodes\u003cbr \/\u003e\nNAME                          STATUS     ROLES    AGE   VERSION\u003cbr \/\u003e\ntoolsbeta-test-k8s-master-1   NotReady   <none>   95m   v1.15.0\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_306\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302020\" id=\"5302020\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_305\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_40\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302020\" data-sigil=\"has-tooltip\" data-meta=\"0_304\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:27:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_302\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_303\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_41\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSomething is flapping, I think.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_315\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302067\" id=\"5302067\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_314\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_42\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302067\" data-sigil=\"has-tooltip\" data-meta=\"0_313\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:39:03 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_311\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_312\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_43\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI think I see the problem.  Kubeadm designated the node by it's short name kubelet.go:2248] node "toolsbeta-test-k8s-master-1" not found.\u003cbr \/\u003e\nPerhaps that's not in the cert for the kubelet?  Checking.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_324\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302118\" id=\"5302118\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_323\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_44\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302118\" data-sigil=\"has-tooltip\" data-meta=\"0_322\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:56:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_320\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_321\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_45\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eNah, that's consistent.  running \u003ctt class=\"remarkup-monospaced\"\u003ekubeadm reset\u003c\/tt\u003e and then \u003ctt class=\"remarkup-monospaced\"\u003ekubeadm init --config\u003c\/tt\u003e again.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_333\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302129\" id=\"5302129\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_332\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_46\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302129\" data-sigil=\"has-tooltip\" data-meta=\"0_331\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 9:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 21:59:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_329\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_330\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_47\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThat seems to have it up consistently.  I re-applied the calico config.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_342\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302182\" id=\"5302182\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_341\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_48\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302182\" data-sigil=\"has-tooltip\" data-meta=\"0_340\"\u003e\u003cspan class=\"screen-only\"\u003eJul 2 2019, 10:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-02 22:47:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_338\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_339\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_49\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eCurrent status: the node never becomes ready because CNI isn't up.  CNI isn't up because the daemonset for calico doesn't actually start running for some reason. daemonset.apps\/calico-node is the daemonset.\u003c\/p\u003e\n\n\u003cp\u003eI did notice it never pulled the image for some reason, so I manually did \u003ctt class=\"remarkup-monospaced\"\u003edocker image pull calico\/cni:v3.8.0\u003c\/tt\u003e and restarted docker and kubelete.  Nothing yet.  Docker has to recognize it has a CNI...and it has to actually spin it up.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_351\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302894\" id=\"5302894\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_350\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_50\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302894\" data-sigil=\"has-tooltip\" data-meta=\"0_349\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 08:58:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_347\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_348\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_51\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T215531#5301722\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_5\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531#5301722\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_7\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eMerging the join, haproxy and firewall patches.  Please feel free to discuss\/fix\/smash them in your time \u003ca href=\"\/p\/aborrero\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_6\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aborrero\u003c\/span\u003e\u003c\/a\u003e if you disagree with any of those approaches.\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI agree with your changes! :-) +1\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_360\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302900\" id=\"5302900\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_359\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_52\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302900\" data-sigil=\"has-tooltip\" data-meta=\"0_358\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:08 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:08:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_356\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_357\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_53\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520388 had a related patch set uploaded (by Arturo Borrero Gonzalez; owner: Arturo Borrero Gonzalez):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: factorice preflight checks to the common profile\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520388\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520388\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_369\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302911\" id=\"5302911\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_368\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_54\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302911\" data-sigil=\"has-tooltip\" data-meta=\"0_367\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:11 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:11:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_365\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_366\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_55\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520388 \u003cstrong\u003emerged\u003c\/strong\u003e by Arturo Borrero Gonzalez:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: factorice preflight checks to the common profile\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520388\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520388\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_378\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302925\" id=\"5302925\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_377\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_56\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302925\" data-sigil=\"has-tooltip\" data-meta=\"0_376\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:19 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:19:34 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_374\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_375\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_57\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520238 abandoned by Arturo Borrero Gonzalez:\u003cbr \/\u003e\ntoolforge: k8s: add calico yaml config\u003c\/p\u003e\n\n\u003cp\u003eReason:\u003cbr \/\u003e\nApplied \u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/c\/operations\/puppet\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/c\/operations\/puppet\/\u003c\/a\u003e \/520233 instead\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520238\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520238\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_387\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302939\" id=\"5302939\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_386\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_58\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302939\" data-sigil=\"has-tooltip\" data-meta=\"0_385\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:22 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:22:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_383\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_384\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_59\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWu3JHS0EHTBTPG-1IwN\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_1\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2019-07-03T09:22:27Z] <arturo> delete all \u003ctt class=\"remarkup-monospaced\"\u003etoolsbeta-arturo-k8s-*\u003c\/tt\u003e instances. We no longer require them per new approach at \u003ca href=\"\/T215531\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_8\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_396\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302972\" id=\"5302972\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_395\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_60\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302972\" data-sigil=\"has-tooltip\" data-meta=\"0_394\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:31 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:31:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_392\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_393\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_61\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520319 \u003cstrong\u003emerged\u003c\/strong\u003e by Arturo Borrero Gonzalez:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: refactor to join a node to the new cluster\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520319\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520319\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_405\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5302976\" id=\"5302976\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_404\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_62\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5302976\" data-sigil=\"has-tooltip\" data-meta=\"0_403\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:32 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:32:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_401\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_402\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_63\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWu3LXa3EHTBTPG-1JQZ\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_2\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2019-07-03T09:32:18Z] <arturo> create \u003ctt class=\"remarkup-monospaced\"\u003etoolsbeta-test-k8s-worker-1\u003c\/tt\u003e VM and a puppet prefix for it (\u003ca href=\"\/T215531\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_9\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_414\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303023\" id=\"5303023\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_413\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_64\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303023\" data-sigil=\"has-tooltip\" data-meta=\"0_412\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:47 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:47:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_410\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_411\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_65\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520397 had a related patch set uploaded (by Arturo Borrero Gonzalez; owner: Arturo Borrero Gonzalez):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: fix half-done code factorization for kubadm_join\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520397\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520397\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_423\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303025\" id=\"5303025\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_422\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_66\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303025\" data-sigil=\"has-tooltip\" data-meta=\"0_421\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 09:49:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_419\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_420\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_67\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520397 \u003cstrong\u003emerged\u003c\/strong\u003e by Arturo Borrero Gonzalez:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: fix half-done code factorization for kubadm_join\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520397\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520397\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_432\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303118\" id=\"5303118\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_431\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_68\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303118\" data-sigil=\"has-tooltip\" data-meta=\"0_430\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 10:24 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 10:24:45 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_428\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_429\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_69\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520406 had a related patch set uploaded (by Arturo Borrero Gonzalez; owner: Arturo Borrero Gonzalez):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: apilb: specify path for systemctl call when reloading config\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520406\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520406\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_441\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303121\" id=\"5303121\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_440\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_70\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303121\" data-sigil=\"has-tooltip\" data-meta=\"0_439\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 10:26 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 10:26:57 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_437\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_438\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_71\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520406 \u003cstrong\u003emerged\u003c\/strong\u003e by Arturo Borrero Gonzalez:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: apilb: specify path for systemctl call when reloading config\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520406\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520406\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_450\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303182\" id=\"5303182\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_449\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_72\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303182\" data-sigil=\"has-tooltip\" data-meta=\"0_448\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 10:48 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 10:48:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_446\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_447\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_73\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYes, we have issues. My first impression is that the cluster is not bootstrapped and kube-apiserver is not running anywhere.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $ kubectl get nodes\nNAME                          STATUS     ROLES    AGE   VERSION\ntoolsbeta-test-k8s-master-1   NotReady   master   36m   v1.15.0\naborrero@toolsbeta-test-k8s-master-1:~ $ kubectl get ns\nNAME              STATUS   AGE\ndefault           Active   37m\nkube-node-lease   Active   37m\nkube-public       Active   37m\nkube-system       Active   37m\naborrero@toolsbeta-test-k8s-master-1:~ $ kubectl get pods -ns=kube-system\nNo resources found.\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThere are plenty of messages like these in the logs:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eJul  3 10:41:18 toolsbeta-test-k8s-master-1 kubelet[10058]: E0703 10:41:18.193441   10058 kubelet.go:1647] Failed creating a mirror pod for "kube-controller-manager-toolsbeta-test-k8s-master-1_kube-system(369b4cdba0a07c0bdf81232f956f0ecd)": pods "kube-controller-manager-toolsbeta-test-k8s-master-1" is forbidden: no providers available to validate pod request\nJul  3 10:41:28 toolsbeta-test-k8s-master-1 kubelet[10058]: E0703 10:41:28.533903   10058 kubelet.go:1647] Failed creating a mirror pod for "etcd-toolsbeta-test-k8s-master-1_kube-system(4fbf23c4994aae3f8711685bb3a1ce28)": pods "etcd-toolsbeta-test-k8s-master-1" is forbidden: no providers available to validate pod request\nJul  3 10:45:22 toolsbeta-test-k8s-master-1 kubelet[10058]: E0703 10:45:22.976843   10058 kubelet.go:1647] Failed creating a mirror pod for "kube-apiserver-toolsbeta-test-k8s-master-1_kube-system(41766b16ea4d86caaf58453a2cb16e3d)": pods "kube-apiserver-toolsbeta-test-k8s-master-1" is forbidden: no providers available to validate pod request\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ewhich suggest apiserver and friends aren't running anywhere. Reason? I don't know, but I will try now in a fresh VM.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_459\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303184\" id=\"5303184\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_458\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_74\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303184\" data-sigil=\"has-tooltip\" data-meta=\"0_457\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 10:49 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 10:49:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_455\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_456\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_75\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWu3dFoiOwpQ-3Pkzd76\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_3\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2019-07-03T10:49:43Z] <arturo> recreate \u003ctt class=\"remarkup-monospaced\"\u003etoolsbeta-test-k8s-master-1\u003c\/tt\u003e VM (\u003ca href=\"\/T215531\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_10\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_468\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303334\" id=\"5303334\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_467\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_76\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303334\" data-sigil=\"has-tooltip\" data-meta=\"0_466\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 11:32 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 11:32:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_464\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_465\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_77\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSomething is wrong in our configuration.\u003c\/p\u003e\n\n\u003cp\u003eI was able to get the cluster working just by using defaults (i.e, ignoring our init config):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\" style=\" max-height: 40em; overflow: auto;\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e sudo kubeadm init\n\u003cspan class=\"go\"\u003e[init] Using Kubernetes version: v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[preflight] Running pre-flight checks\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[preflight] Pulling images required for setting up a Kubernetes cluster\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[preflight] This might take a minute or two, depending on the speed of your internet connection\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Writing kubelet environment file with flags to file "\/var\/lib\/kubelet\/kubeadm-flags.env"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Writing kubelet configuration to file "\/var\/lib\/kubelet\/config.yaml"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Activating the kubelet service\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Using certificateDir folder "\/etc\/kubernetes\/pki"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "ca" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "apiserver" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] apiserver serving cert is signed for DNS names [toolsbeta-test-k8s-master-1 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 172.16.0.104]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "apiserver-kubelet-client" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "etcd\/ca" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "etcd\/server" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] etcd\/server serving cert is signed for DNS names [toolsbeta-test-k8s-master-1 localhost] and IPs [172.16.0.104 127.0.0.1 ::1]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "etcd\/peer" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] etcd\/peer serving cert is signed for DNS names [toolsbeta-test-k8s-master-1 localhost] and IPs [172.16.0.104 127.0.0.1 ::1]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "etcd\/healthcheck-client" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "apiserver-etcd-client" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "front-proxy-ca" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "front-proxy-client" certificate and key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[certs] Generating "sa" key and public key\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubeconfig] Using kubeconfig folder "\/etc\/kubernetes"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubeconfig] Writing "admin.conf" kubeconfig file\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubeconfig] Writing "kubelet.conf" kubeconfig file\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubeconfig] Writing "controller-manager.conf" kubeconfig file\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubeconfig] Writing "scheduler.conf" kubeconfig file\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[control-plane] Using manifest folder "\/etc\/kubernetes\/manifests"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[control-plane] Creating static Pod manifest for "kube-apiserver"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[control-plane] Creating static Pod manifest for "kube-controller-manager"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[control-plane] Creating static Pod manifest for "kube-scheduler"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[etcd] Creating static Pod manifest for local etcd in "\/etc\/kubernetes\/manifests"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "\/etc\/kubernetes\/manifests". This can take up to 4m0s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[apiclient] All control plane components are healthy after 21.005768 seconds\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet] Creating a ConfigMap "kubelet-config-1.15" in namespace kube-system with the configuration for the kubelets in the cluster\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[upload-certs] Skipping phase. Please see --upload-certs\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[mark-control-plane] Marking the node toolsbeta-test-k8s-master-1 as control-plane by adding the label "node-role.kubernetes.io\/master=''"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[mark-control-plane] Marking the node toolsbeta-test-k8s-master-1 as control-plane by adding the taints [node-role.kubernetes.io\/master:NoSchedule]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[bootstrap-token] Using token: bmvsr4.23fnne16ttoab7lh\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[addons] Applied essential addon: CoreDNS\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[addons] Applied essential addon: kube-proxy\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eYour Kubernetes control-plane has initialized successfully!\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eTo start using your cluster, you need to run the following as a regular user:\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003e  mkdir -p $HOME\/.kube\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e  sudo cp -i \/etc\/kubernetes\/admin.conf $HOME\/.kube\/config\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e  sudo chown $(id -u):$(id -g) $HOME\/.kube\/config\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eYou should now deploy a pod network to the cluster.\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eRun "kubectl apply -f [podnetwork].yaml" with one of the options listed at:\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e  https:\/\/kubernetes.io\/docs\/concepts\/cluster-administration\/addons\/\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eThen you can join any number of worker nodes by running the following on each as root:\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003ekubeadm join 172.16.0.104:6443 --token bmvsr4.23fnne16ttoab7lh \\\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e    --discovery-token-ca-cert-hash sha256:db141224ed4e6a89c54ae2eaf64b9bc4163c55d71455a15b659a334a92357cfd \u003c\/span\u003e\n\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e sudo cp -i \/etc\/kubernetes\/admin.conf \u003cspan class=\"nv\"\u003e$HOME\u003c\/span\u003e\/.kube\/config\n\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e sudo chown \u003cspan class=\"k\"\u003e$(\u003c\/span\u003eid -u\u003cspan class=\"k\"\u003e)\u003c\/span\u003e:\u003cspan class=\"k\"\u003e$(\u003c\/span\u003eid -g\u003cspan class=\"k\"\u003e)\u003c\/span\u003e \u003cspan class=\"nv\"\u003e$HOME\u003c\/span\u003e\/.kube\/config\n\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e kubectl get nodes\n\u003cspan class=\"go\"\u003eNAME                          STATUS     ROLES    AGE   VERSION\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-1   NotReady   master   31s   v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u230269% aborrero@toolsbeta-test-k8s-master-1:~ $ kubectl get pods --all-namespaces\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eNAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   coredns-5c98db65d4-4rfj6   0\/1     Pending   0          27s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   coredns-5c98db65d4-wws2c   0\/1     Pending   0          27s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-proxy-lq9sk           1\/1     Running   0          27s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e\u230263% aborrero@toolsbeta-test-k8s-master-1:~ $ sudo kubectl apply -f \/etc\/kubernetes\/calico.yaml \u003c\/span\u003e\n\u003cspan class=\"go\"\u003econfigmap\/calico-config created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/felixconfigurations.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/ipamblocks.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/blockaffinities.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/ipamhandles.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/ipamconfigs.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/bgppeers.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/bgpconfigurations.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/ippools.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/hostendpoints.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/clusterinformations.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/globalnetworkpolicies.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/globalnetworksets.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/networkpolicies.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecustomresourcedefinition.apiextensions.k8s.io\/networksets.crd.projectcalico.org created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eclusterrole.rbac.authorization.k8s.io\/calico-kube-controllers created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eclusterrolebinding.rbac.authorization.k8s.io\/calico-kube-controllers created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eclusterrole.rbac.authorization.k8s.io\/calico-node created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eclusterrolebinding.rbac.authorization.k8s.io\/calico-node created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003edaemonset.apps\/calico-node created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eserviceaccount\/calico-node created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003edeployment.apps\/calico-kube-controllers created\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eserviceaccount\/calico-kube-controllers created\u003c\/span\u003e\n\n\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e kubectl get pods --all-namespaces\n\u003cspan class=\"go\"\u003eNAMESPACE     NAME                                                  READY   STATUS    RESTARTS   AGE\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-kube-controllers-59f54d6bbc-gl5lt              0\/1     Running   0          30s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-node-wrg5h                                     1\/1     Running   0          30s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   coredns-5c98db65d4-4rfj6                              0\/1     Running   0          89s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   coredns-5c98db65d4-wws2c                              1\/1     Running   0          89s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   etcd-toolsbeta-test-k8s-master-1                      1\/1     Running   0          20s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-apiserver-toolsbeta-test-k8s-master-1            1\/1     Running   0          29s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-controller-manager-toolsbeta-test-k8s-master-1   1\/1     Running   0          36s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-proxy-lq9sk                                      1\/1     Running   0          89s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-scheduler-toolsbeta-test-k8s-master-1            1\/1     Running   0          44s\u003c\/span\u003e\n\n\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e kubectl get nodes\n\u003cspan class=\"go\"\u003eNAME                          STATUS   ROLES    AGE    VERSION\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-1   Ready    master   102s   v1.15.0\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_477\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303556\" id=\"5303556\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_476\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_78\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303556\" data-sigil=\"has-tooltip\" data-meta=\"0_475\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 1:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 13:01:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_473\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_474\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_79\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520429 had a related patch set uploaded (by Bstorm; owner: Bstorm):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: correct the values for bootstrapping a cluster\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520429\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520429\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_486\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303645\" id=\"5303645\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_485\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_80\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303645\" data-sigil=\"has-tooltip\" data-meta=\"0_484\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 1:38 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 13:38:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_482\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_483\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_81\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWhat is dogging us (besides the podsecuritypolicy fun) is the pod IP settings.  They have to all within the -cluster-cidr.  I found where that damned setting is.  It's on the kube-controller-manager:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e--cluster-cidr string\n\tCIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eI'll get the defaults and what we need to change it to.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_495\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303646\" id=\"5303646\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_494\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_82\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303646\" data-sigil=\"has-tooltip\" data-meta=\"0_493\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 1:39 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 13:39:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_491\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_492\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_83\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt seems to end up set to the same value as our podSubnet: \u003ctt class=\"remarkup-monospaced\"\u003e    - --cluster-cidr=192.168.0.0\/24\u003c\/tt\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_504\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303651\" id=\"5303651\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_503\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_84\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303651\" data-sigil=\"has-tooltip\" data-meta=\"0_502\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 1:43 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 13:43:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_500\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_501\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_85\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt's recommended that the service ips not even come close to overlap and even stay at the default 10.96.whatever.  Helpful hints here as well: \u003ca href=\"https:\/\/docs.projectcalico.org\/v3.8\/getting-started\/kubernetes\/installation\/config-options\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/docs.projectcalico.org\/v3.8\/getting-started\/kubernetes\/installation\/config-options\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_513\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303667\" id=\"5303667\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_512\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_86\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303667\" data-sigil=\"has-tooltip\" data-meta=\"0_511\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 1:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 13:51:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_509\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_510\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_87\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003enoticed this:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eE0703 13:49:37.284017       1 driver-call.go:267] Failed to unmarshal output for command: init, output: "2019\/07\/03 13:49:37 Unix syslog delivery error\\n", error: invalid character '\/' after top-level value\nW0703 13:49:37.284186       1 driver-call.go:150] FlexVolume: driver call failed: executable: \/usr\/libexec\/kubernetes\/kubelet-plugins\/volume\/exec\/nodeagent~uds\/uds, args: [init], error: exit status 1, output: "2019\/07\/03 13:49:37 Unix syslog delivery error\\n"\nE0703 13:49:37.284283       1 plugins.go:746] Error dynamically probing plugins: Error creating Flexvolume plugin from directory nodeagent~uds, skipping. Error: invalid character '\/' after top-level value\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ePossibly for later.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_522\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303680\" id=\"5303680\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_521\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_88\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303680\" data-sigil=\"has-tooltip\" data-meta=\"0_520\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 2:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 14:06:07 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_518\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_519\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_89\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAh, I noticed a cidr mask was added in the local config for the controller manager, which did not match other values either.  Removed that.  Overall, I think the serviceSubnet was a big problem.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_531\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303687\" id=\"5303687\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_530\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_90\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303687\" data-sigil=\"has-tooltip\" data-meta=\"0_529\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 2:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 14:09:46 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_527\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_528\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_91\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWatching the pods with my latest version....it seems stable.  Things have been running for 3m with pods still live.  I won't declare victory yet...\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_540\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303690\" id=\"5303690\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_539\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_92\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303690\" data-sigil=\"has-tooltip\" data-meta=\"0_538\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 2:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 14:10:54 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_536\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_537\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_93\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYeah, I'm not quite sure coredns and the scheduler are happy with everything yet\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_549\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303706\" id=\"5303706\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_548\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_94\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303706\" data-sigil=\"has-tooltip\" data-meta=\"0_547\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 2:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 14:24:15 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_545\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_546\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_95\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe logs of the scheduler showed it was affected by \u003ca href=\"https:\/\/github.com\/kubernetes\/kubeadm\/issues\/1285\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/kubernetes\/kubeadm\/issues\/1285\u003c\/a\u003e.  Trying setting those as well.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_558\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303736\" id=\"5303736\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_557\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_96\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303736\" data-sigil=\"has-tooltip\" data-meta=\"0_556\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 2:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 14:56:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_554\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_555\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_97\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAlso for later, some cgroup\/slice issues related to systemd:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eJul  3 14:55:41 toolsbeta-test-k8s-master-1 kubelet[1398]: W0703 14:55:41.419748    1398 watcher.go:87] Error while processing event ("\/sys\/fs\/cgroup\/memory\/libcontainer_7779_systemd_test_default.slice": 0x40000100 == IN_CREATE|IN_ISDIR): readdirent: no such file or directory\nJul  3 14:55:41 toolsbeta-test-k8s-master-1 kubelet[1398]: W0703 14:55:41.419817    1398 watcher.go:87] Error while processing event ("\/sys\/fs\/cgroup\/devices\/libcontainer_7779_systemd_test_default.slice": 0x40000100 == IN_CREATE|IN_ISDIR): inotify_add_watch \/sys\/fs\/cgroup\/devices\/libcontainer_7779_systemd_test_default.slice: no such file or directory\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_567\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303750\" id=\"5303750\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_566\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_98\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303750\" data-sigil=\"has-tooltip\" data-meta=\"0_565\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 2:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 14:59:12 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_563\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_564\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_99\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWorking baseline kubeadm-init.yaml file:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eapiVersion: kubeadm.k8s.io\/v1beta2\nkind: ClusterConfiguration\nkubernetesVersion: stable\ncontrolPlaneEndpoint: "toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443"\nnetworking:\n  podSubnet: 192.168.0.0\/16\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_576\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5303756\" id=\"5303756\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_575\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_100\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5303756\" data-sigil=\"has-tooltip\" data-meta=\"0_574\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 3:00 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 15:00:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_572\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_573\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_101\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnd that results in this config after merging in kubeadm:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eapiVersion: v1\ndata:\n  ClusterConfiguration: |\n    apiServer:\n      extraArgs:\n        authorization-mode: Node,RBAC\n      timeoutForControlPlane: 4m0s\n    apiVersion: kubeadm.k8s.io\/v1beta2\n    certificatesDir: \/etc\/kubernetes\/pki\n    clusterName: kubernetes\n    controlPlaneEndpoint: toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443\n    controllerManager: {}\n    dns:\n      type: CoreDNS\n    etcd:\n      local:\n        dataDir: \/var\/lib\/etcd\n    imageRepository: k8s.gcr.io\n    kind: ClusterConfiguration\n    kubernetesVersion: v1.15.0\n    networking:\n      dnsDomain: cluster.local\n      podSubnet: 192.168.0.0\/16\n      serviceSubnet: 10.96.0.0\/12\n    scheduler: {}\n  ClusterStatus: |\n    apiEndpoints:\n      toolsbeta-test-k8s-master-1:\n        advertiseAddress: 172.16.0.104\n        bindPort: 6443\n    apiVersion: kubeadm.k8s.io\/v1beta2\n    kind: ClusterStatus\nkind: ConfigMap\nmetadata:\n  creationTimestamp: "2019-07-03T14:52:37Z"\n  name: kubeadm-config\n  namespace: kube-system\n  resourceVersion: "155"\n  selfLink: \/api\/v1\/namespaces\/kube-system\/configmaps\/kubeadm-config\n  uid: 8f54b84d-51a4-4f91-b80b-77dc4b3acbd5\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnything in that should also work (theoretically)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_585\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304209\" id=\"5304209\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_584\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_102\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304209\" data-sigil=\"has-tooltip\" data-meta=\"0_583\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 4:53 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 16:53:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_581\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_582\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_103\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T215531#5303736\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_11\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531#5303736\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/aborrero\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_12\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aborrero\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eAlso for later, some cgroup\/slice issues related to systemd:\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI think this is a race condition between kubelet and systemd.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eJul  3 15:27:58 toolsbeta-test-k8s-master-1 systemd[1]: Removed slice libcontainer_885_systemd_test_default.slice.\nJul  3 15:27:58 toolsbeta-test-k8s-master-1 kubelet[31002]: W0703 15:27:58.410865   31002 watcher.go:87] Error while processing event ("\/sys\/fs\/cgroup\/devices\/libcontainer_885_systemd_test_default.slice": 0x40000100 == IN_CREATE|IN_ISDIR): readdirent: no such file or directory\n[..]\nJul  3 15:28:00 toolsbeta-test-k8s-master-1 systemd[1]: Removed slice libcontainer_1003_systemd_test_default.slice.\nJul  3 15:28:00 toolsbeta-test-k8s-master-1 kubelet[31002]: W0703 15:28:00.338298   31002 watcher.go:87] Error while processing event ("\/sys\/fs\/cgroup\/devices\/libcontainer_1003_systemd_test_default.slice": 0x40000100 == IN_CREATE|IN_ISDIR): inotify_add_watch \/sys\/fs\/cgroup\/devices\/libcontainer_1003_systemd_test_default.slice: no such file or directory\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo systemd deletes the slice, and kubelet is monitoring somehow the event, and the slice vanished before kubelet could handle it.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_594\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304576\" id=\"5304576\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_593\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_104\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304576\" data-sigil=\"has-tooltip\" data-meta=\"0_592\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 6:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 18:37:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_590\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_591\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_105\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520429 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: correct the values for bootstrapping a cluster\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520429\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520429\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_603\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304619\" id=\"5304619\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_602\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_106\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304619\" data-sigil=\"has-tooltip\" data-meta=\"0_601\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 7:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 19:01:31 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_599\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_600\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_107\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSaved the current working config aside (overwriting the .working file) and enabling puppet on the master.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_612\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304659\" id=\"5304659\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_611\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_108\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304659\" data-sigil=\"has-tooltip\" data-meta=\"0_610\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 7:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 19:07:21 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_608\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_609\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_109\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe first master is now using the puppet config and is running fine.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_621\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304751\" id=\"5304751\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_620\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_110\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304751\" data-sigil=\"has-tooltip\" data-meta=\"0_619\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 7:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 19:44:13 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_617\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_618\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_111\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520516 had a related patch set uploaded (by Bstorm; owner: Bstorm):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: set up cert distribution for additional control plane nodes\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520516\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520516\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_630\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304770\" id=\"5304770\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_629\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_112\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304770\" data-sigil=\"has-tooltip\" data-meta=\"0_628\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 20:02:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_626\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_627\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_113\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520516 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: set up cert distribution for additional control plane nodes\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520516\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520516\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_639\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304776\" id=\"5304776\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_638\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_114\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304776\" data-sigil=\"has-tooltip\" data-meta=\"0_637\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 20:12:36 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_635\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_636\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_115\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520519 had a related patch set uploaded (by Bstorm; owner: Bstorm):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: create etcd dir for the certs\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520519\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520519\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_648\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304780\" id=\"5304780\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_647\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_116\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304780\" data-sigil=\"has-tooltip\" data-meta=\"0_646\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 20:19:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_644\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_645\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_117\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 520519 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: create etcd dir for the certs\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/520519\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/520519\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_657\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304868\" id=\"5304868\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_656\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_118\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304868\" data-sigil=\"has-tooltip\" data-meta=\"0_655\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:46 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 20:46:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_653\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_654\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_119\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSo the join config for control plane nodes is basically useless.  \u003cbr \/\u003e\n\u003ca href=\"https:\/\/github.com\/kubernetes\/kubeadm\/issues\/1485\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/kubernetes\/kubeadm\/issues\/1485\u003c\/a\u003e makes it clear that we have to use the command line.  Luckily, not much is required for that if you figure the certs are distributed by puppet.  The CLI is:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003esudo kubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443 --token <token from the join file> --control-plane --discovery-token-unsafe-skip-ca-verification\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eIt worked perfectly.  That said, the last option is only needed if we leave out the ca-verification token.  We honestly arguably don't need it unless we are concerned about someone spoofing the master on us?  It isn't really best practice, but we can sort that out as we go if we think we need it later.  It is only for joining control plane nodes.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_666\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304901\" id=\"5304901\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_665\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_120\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304901\" data-sigil=\"has-tooltip\" data-meta=\"0_664\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 20:56:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_662\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_663\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_121\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWith the second node up, I am able to confirm that the LB was able to reload and all that right from the hiera change :)\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"console\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan class=\"gp\"\u003e$ echo "show stat" | socat \/run\/haproxy\/haproxy.sock stdio\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,agent_status,agent_code,agent_duration,check_desc,agent_desc,check_rise,check_fall,check_health,agent_rise,agent_fall,agent_health,addr,cookie,mode,algo,conn_rate,conn_rate_max,conn_tot,intercepted,dcon,dses,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,2,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,,,,,,,,,,,,,,tcp,,0,0,0,,0,0,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,****toolsbeta-test-k8s-master-1,****0,0,0,0,250,0,0,0,,0,,0,0,0,0,UP,100,1,0,0,0,14,0,256,1,3,1,,0,,2,0,,0,L4OK,,0,,,,,,,,,,,0,0,,,,,-1,,,0,0,0,0,,,,Layer4 check passed,,2,2,3,,,,,,tcp,,,,,,,,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,****toolsbeta-test-k8s-master-2****,0,0,0,0,250,0,0,0,,0,,0,0,0,0,UP,100,1,0,0,0,14,0,256,1,3,2,,0,,2,0,,0,L4OK,,2,,,,,,,,,,,0,0,,,,,-1,,,0,0,0,0,,,,Layer4 check passed,,2,2,3,,,,,,tcp,,,,,,,,\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ek8s-api,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,200,2,0,,0,14,0,,1,3,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,,,,,,,,,,,,,,tcp,,,,,,,,\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003easterisks added so the server names stand out :-p\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_675\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304907\" id=\"5304907\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_674\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_122\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304907\" data-sigil=\"has-tooltip\" data-meta=\"0_673\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 8:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 20:59:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_671\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_672\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_123\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI am adding \u003ctt class=\"remarkup-monospaced\"\u003eprofile::toolforge::k8s::existing_certs: true\u003c\/tt\u003e on the instance puppet config in horizon so that the certs are distributed only to the new masters.  It probably wouldn't break anything to put it in the prefix, but honestly, I think it is reasonable to just document for rebuilding controlplane nodes.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_684\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304916\" id=\"5304916\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_683\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_124\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304916\" data-sigil=\"has-tooltip\" data-meta=\"0_682\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 21:01:32 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_680\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_681\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_125\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAnother good reason to do it on the instance: on the initial puppet run against labspuppetmaster, it'd fail because the secrets would be on the standalone :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_693\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304978\" id=\"5304978\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_692\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_126\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304978\" data-sigil=\"has-tooltip\" data-meta=\"0_691\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:16 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 21:16:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_689\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_690\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_127\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@toolsbeta-test-k8s-master-3:~# kubectl get nodes\nNAME                          STATUS   ROLES    AGE    VERSION\ntoolsbeta-test-k8s-master-1   Ready    master   124m   v1.15.0\ntoolsbeta-test-k8s-master-2   Ready    master   28m    v1.15.0\ntoolsbeta-test-k8s-master-3   Ready    master   35s    v1.15.0\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSo we still need a worker node (which should be deployable with a simple \u003ctt class=\"remarkup-monospaced\"\u003ekubeadm join --config \/etc\/kubernetes\/kubeadm-join.yaml\u003c\/tt\u003e).  Then we have to work out deploying that little service I set up as a webhook.  PodSecurityPolicy stuff (which might be something we apply after kubeadm runs init?).  maintain-kubeusers the x.509+RBAC edition (or something else like Dex, but it seems like x.509s might be easy with k8s acting as CA).  Ingress that works with kube-proxy or directly with webservice somehow. But we might really be beyond the basic deployment method now!!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_702\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5304981\" id=\"5304981\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_701\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_128\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5304981\" data-sigil=\"has-tooltip\" data-meta=\"0_700\"\u003e\u003cspan class=\"screen-only\"\u003eJul 3 2019, 9:23 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-03 21:23:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_698\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_699\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_129\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eAs another note: we should make sure that the final form of this has a floating IP for the LB to provide a certain sort of redundancy where we can always move it to a new server.  All stuff to get on a wiki page.  Stacked control plane works great though, now! :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_711\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5306564\" id=\"5306564\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_710\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_130\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5306564\" data-sigil=\"has-tooltip\" data-meta=\"0_709\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 12:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 12:36:53 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_707\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_708\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_131\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI confirm that joining a working node just works:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-worker-1:~ $\u003c\/span\u003e sudo kubeadm join --config \/etc\/kubernetes\/kubeadm-join.yaml \n\u003cspan class=\"go\"\u003e[preflight] Running pre-flight checks\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[preflight] Reading configuration from the cluster...\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.15" ConfigMap in the kube-system namespace\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Writing kubelet configuration to file "\/var\/lib\/kubelet\/config.yaml"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Writing kubelet environment file with flags to file "\/var\/lib\/kubelet\/kubeadm-flags.env"\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Activating the kubelet service\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eThis node has joined the cluster:\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e* Certificate signing request was sent to apiserver and a response was received.\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e* The Kubelet was informed of the new secure connection details.\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eRun 'kubectl get nodes' on the control-plane to see this node join the cluster.\u003c\/span\u003e\n\n\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-master-1:~ $\u003c\/span\u003e kubectl get nodes\n\u003cspan class=\"go\"\u003eNAME                          STATUS   ROLES    AGE     VERSION\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-1   Ready    master   17h     v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-2   Ready    master   15h     v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-3   Ready    master   15h     v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-worker-1   Ready    <none>   5m33s   v1.15.0\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_720\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5306644\" id=\"5306644\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_719\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_132\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5306644\" data-sigil=\"has-tooltip\" data-meta=\"0_718\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 1:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 13:09:10 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_716\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_717\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_133\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI've been trying to re-create the cluster with the certificate dancing. I didn't like it, because it involves storing the cert data into the labs\/private repo.\u003c\/p\u003e\n\n\u003cp\u003eI think kubeadm has an option for this, which I just tested, and which is:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cdiv class=\"remarkup-code-header\"\u003efirst_control_plane_node\u003c\/div\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-master-1:~#\u003c\/span\u003e kubeadm init --config \/etc\/kubernetes\/kubeadm-init.yaml --upload-certs \u003cspan class=\"o\"\u003e&&\u003c\/span\u003e cp \/etc\/kubernetes\/admin.conf \/root\/.kube\/config\n\u003cspan class=\"go\"\u003e[..]\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eYour Kubernetes control-plane has initialized successfully!\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eTo start using your cluster, you need to run the following as a regular user:\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003e  mkdir -p $HOME\/.kube\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e  sudo cp -i \/etc\/kubernetes\/admin.conf $HOME\/.kube\/config\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e  sudo chown $(id -u):$(id -g) $HOME\/.kube\/config\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eYou should now deploy a pod network to the cluster.\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eRun "kubectl apply -f [podnetwork].yaml" with one of the options listed at:\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e  https:\/\/kubernetes.io\/docs\/concepts\/cluster-administration\/addons\/\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eYou can now join any number of the control-plane node running the following command on each as root:\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003e  kubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443 --token m7uakr.ern5lmlpv7gnkacw \\\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e    --discovery-token-ca-cert-hash sha256:5dcfa8d5992b65759daac3bc142e7e2f2f5ee584c2a40e4dab5c82a39fa89df6 \\\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e    --experimental-control-plane --certificate-key bb492a5fdc9dba43344a7c7fbae0767d8e41b5b7b5abd4d47177ed24c108c385\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003ePlease note that the certificate-key gives access to cluster sensitive data, keep it secret!\u003c\/span\u003e\n\u003cspan class=\"go\"\u003eAs a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use \u003c\/span\u003e\n\u003cspan class=\"go\"\u003e"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003eThen you can join any number of worker nodes by running the following on each as root:\u003c\/span\u003e\n\n\u003cspan class=\"go\"\u003ekubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443 --token m7uakr.ern5lmlpv7gnkacw \\\u003c\/span\u003e\n\u003cspan class=\"go\"\u003e    --discovery-token-ca-cert-hash sha256:5dcfa8d5992b65759daac3bc142e7e2f2f5ee584c2a40e4dab5c82a39fa89df6 \u003c\/span\u003e\n\n\n\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-master-1:~#\u003c\/span\u003e kubectl apply -f \/etc\/kubernetes\/calico.yaml\n\u003cspan class=\"go\"\u003e[..]\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cdiv class=\"remarkup-code-header\"\u003esecond_control_plane_node\u003c\/div\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-master-2:~#\u003c\/span\u003e kubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443 --token m7uakr.ern5lmlpv7gnkacw     --discovery-token-ca-cert-hash sha256:5dcfa8d5992b65759daac3bc142e7e2f2f5ee584c2a40e4dab5c82a39fa89df6     --experimental-control-plane --certificate-key bb492a5fdc9dba43344a7c7fbae0767d8e41b5b7b5abd4d47177ed24c108c385\n\u003cspan class=\"go\"\u003e[...]\u003c\/span\u003e\n\u003cspan class=\"gp gp-VirtualEnv\"\u003e(works fine)\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAnd also works fine for the worker node:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cdiv class=\"remarkup-code-header\"\u003eworker_node\u003c\/div\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-worker-1:~#\u003c\/span\u003e kubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443 --token m7uakr.ern5lmlpv7gnkacw --discovery-token-ca-cert-hash sha256:5dcfa8d5992b65759daac3bc142e7e2f2f5ee584c2a40e4dab5c82a39fa89df6 \n\u003cspan class=\"go\"\u003e[..]\u003c\/span\u003e\n\u003cspan class=\"gp gp-VirtualEnv\"\u003e(works fine)\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cdiv class=\"remarkup-code-header\"\u003eresults\u003c\/div\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-master-1:~#\u003c\/span\u003e kubectl get pods --all-namespaces\n\u003cspan class=\"go\"\u003eNAMESPACE     NAME                                                  READY   STATUS             RESTARTS   AGE\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-kube-controllers-59f54d6bbc-nzslj              1\/1     Running            1          10m\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-node-4vmhx                                     1\/1     Running            0          10m\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-node-hkwmg                                     1\/1     Running            0          8m50s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-node-ltfhh                                     1\/1     Running            0          9m52s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   calico-node-vjvpr                                     1\/1     Running            0          8m57s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   coredns-5c98db65d4-dld6z                              1\/1     Running            0          10m\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   coredns-5c98db65d4-rglrk                              1\/1     Running            0          10m\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   etcd-toolsbeta-test-k8s-master-1                      1\/1     Running            0          9m30s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   etcd-toolsbeta-test-k8s-master-2                      1\/1     Running            3          9m51s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   etcd-toolsbeta-test-k8s-master-3                      1\/1     Running            0          8m55s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-apiserver-toolsbeta-test-k8s-master-1            1\/1     Running            0          9m28s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-apiserver-toolsbeta-test-k8s-master-2            0\/1     CrashLoopBackOff   4          9m52s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-apiserver-toolsbeta-test-k8s-master-3            1\/1     Running            1          8m52s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-controller-manager-toolsbeta-test-k8s-master-1   1\/1     Running            1          9m37s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-controller-manager-toolsbeta-test-k8s-master-2   1\/1     Running            0          9m52s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-controller-manager-toolsbeta-test-k8s-master-3   1\/1     Running            0          8m3s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-proxy-6jvj7                                      1\/1     Running            0          10m\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-proxy-848mj                                      1\/1     Running            0          9m52s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-proxy-dqgbs                                      1\/1     Running            0          8m57s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-proxy-jv47p                                      1\/1     Running            0          8m50s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-scheduler-toolsbeta-test-k8s-master-1            1\/1     Running            1          9m37s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-scheduler-toolsbeta-test-k8s-master-2            1\/1     Running            0          9m52s\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ekube-system   kube-scheduler-toolsbeta-test-k8s-master-3            1\/1     Running            0          8m6s\u003c\/span\u003e\n\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-master-1:~#\u003c\/span\u003e kubectl get nodes\n\u003cspan class=\"go\"\u003eNAME                          STATUS   ROLES    AGE     VERSION\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-1   Ready    master   10m     v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-2   Ready    master   9m54s   v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-master-3   Ready    master   8m59s   v1.15.0\u003c\/span\u003e\n\u003cspan class=\"go\"\u003etoolsbeta-test-k8s-worker-1   Ready    <none>   8m52s   v1.15.0\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eHonestly the native kubeadm mechanism for distributing the certs seems great! and easy to use. And allows us to avoid the back-and-forth to the labs\/private repo. What do you think?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_730\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5306869\" id=\"5306869\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_729\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_134\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#5306869\" data-sigil=\"has-tooltip\" data-meta=\"0_728\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 2:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 14:27:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_726\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_727\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_135\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMy reason for avoiding the upload-certs option is that they are self-destructing.  The secret that contains them lasts 24 hours by default, and then you cannot rebuild a control plane node.  If we keep our certs in a secret (which may be kind of clunky at best, but it is our supported mechanism for maintaining secrets) then we can rebuild a control plane node whenever needed.  We won't be wiping the nodes regularly.  I think a rebuild of one node seems more likely to me than a full cluster wipe after deployment; even a healthy thing to do now and and again.  To do with via the native mechanism, the certs need to be manually copied.\u003cbr \/\u003e\nRandom note: \u003ctt class=\"remarkup-monospaced\"\u003e--experimental-control-plane \u003c\/tt\u003e is deprecated in this version.  It's just \u003ctt class=\"remarkup-monospaced\"\u003e--control-plane\u003c\/tt\u003e now--but they do the same thing.  It's funny that they haven't updated the output yet. :)\u003c\/p\u003e\n\n\u003cp\u003eThe self-destruct is something that is configurable, but how long do we want the encrypted secret floating around in k8s?  I mean, it is an encrypted secret, so it's not that bad--arguably better than what we usually do!  Maybe we can check into that? However, since we will very rarely ever build a control plane, but we might actually rebuild a single node of the control plane, I think it is not helpful to make it easy to deploy a complete cluster vs making it easy to deploy a single node that can join quickly in our use case.\u003c\/p\u003e\n\n\u003cp\u003eWe could also script the process of uploading the certs for the "current tools k8s cluster" so that the process is less bleh (but it'll still require a local commit unless we just decide to do more management of encrypted secrets directly in k8s).\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_739\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5306882\" id=\"5306882\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_738\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_136\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5306882\" data-sigil=\"has-tooltip\" data-meta=\"0_737\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 2:32 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 14:32:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_735\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_736\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_137\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eLemme know what you think! I kind of struggled with the whole idea myself, and that's where I ended up.  The logic could possibly be turned in the opposite direction and one could say we need a script for rebuilding a node (that copies the certs for us) instead, figuring *that* won't happen often either?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_748\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5306950\" id=\"5306950\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_747\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_138\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5306950\" data-sigil=\"has-tooltip\" data-meta=\"0_746\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 2:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 14:57:01 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_744\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_745\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_139\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI think we can use this kubeadm option:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003esudo kubeadm init phase upload-certs --upload-certs\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ewhich just re-upload the certs. So the workflow I'm proposing is:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ebootstrap the first control plane node (master-1)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ebootstrap the other control plane nodes (master-2, master-3)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003edays or weeks pass\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eif we later want to rebuild any of the 3, use the command above to re-generate the certs into the secret store\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ebootstrap the new master node the same way as before, because certs are already in the secret store.\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_753\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5307014\" id=\"5307014\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_750\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_140\"\u003eaborrero\u003c\/a\u003e mentioned this in \u003ca href=\"\/T215530\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_141\"\u003eT215530: Sort out the best method of spinning up multiple toolforge kubernetes masters\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307014\" data-sigil=\"has-tooltip\" data-meta=\"0_749\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 3:41 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 15:41:29 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-shield phui-timeline-icon\" data-meta=\"0_751\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_142\"\u003eaborrero\u003c\/a\u003e closed subtask \u003ca href=\"\/T215529\" class=\"phui-handle handle-status-closed\"\u003eT215529: Puppetize\/stand up a load balancer for K8s API servers\u003c\/a\u003e as \u003cspan class=\"phui-timeline-value\"\u003eResolved\u003c\/span\u003e.\u003c\/div\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_752\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_143\"\u003eaborrero\u003c\/a\u003e mentioned this in \u003ca href=\"\/T215529\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_144\"\u003eT215529: Puppetize\/stand up a load balancer for K8s API servers\u003c\/a\u003e.\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_756\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5307052\" id=\"5307052\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_755\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_145\"\u003eaborrero\u003c\/a\u003e mentioned this in \u003ca href=\"\/T215663\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_146\"\u003eT215663: Stand up upgraded Toolforge etcd clusters\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307052\" data-sigil=\"has-tooltip\" data-meta=\"0_754\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 3:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 15:45:04 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_766\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5307104\" id=\"5307104\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_765\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_147\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#5307104\" data-sigil=\"has-tooltip\" data-meta=\"0_764\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 4:33 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 16:33:44 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_762\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_763\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_148\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T215531#5306950\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_13\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531#5306950\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/aborrero\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_16\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@aborrero\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eI think we can use this kubeadm option:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003esudo kubeadm init phase upload-certs --upload-certs\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eI just tested this. It works!\u003c\/p\u003e\n\n\u003cp\u003eSince the very basic deployment is already working in toolsbeta, I would suggest we split the remaining work into subtasks, like:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDesign and document the concrete workflow\/steps for bootstrapping the k8s cluster\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDesign and document how to integrate the cluster with PodSecurityPolicy \u003ca href=\"\/T227290\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_14\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT227290: Design and document how to integrate the new Toolforge k8s cluster with PodSecurityPolicy\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eDesign the integration with other custom admission controllers \u003ca href=\"\/T215678\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_15\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215678: Replace each of the custom controllers with something in a new Toolforge Kubernetes setup\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eHow Toolforge users could interact with the new cluster\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cp\u003eI think I will create those as subtasks.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_769\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5307110\" id=\"5307110\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_768\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_149\"\u003eaborrero\u003c\/a\u003e mentioned this in \u003ca href=\"\/T169287\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_150\"\u003eT169287: etcd config depends on puppet certs, but puppet doesn't know\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307110\" data-sigil=\"has-tooltip\" data-meta=\"0_767\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 4:37 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 16:37:08 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_772\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5307120\" id=\"5307120\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_771\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_151\"\u003eaborrero\u003c\/a\u003e mentioned this in \u003ca href=\"\/T142862\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_152\"\u003eT142862: Setup Kubernetes Masters in a HA setup\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307120\" data-sigil=\"has-tooltip\" data-meta=\"0_770\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 4:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 16:42:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_775\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-minor-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003ca name=\"5307145\" id=\"5307145\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-link phui-timeline-icon\" data-meta=\"0_774\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_153\"\u003eaborrero\u003c\/a\u003e mentioned this in \u003ca href=\"\/T215679\" class=\"phui-handle handle-status-closed\" data-sigil=\"hovercard\" data-meta=\"0_154\"\u003eT215679: Sort out and test deploying the worker nodes in a sane fashion\u003c\/a\u003e.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307145\" data-sigil=\"has-tooltip\" data-meta=\"0_773\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 5:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 17:06:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_784\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5307161\" id=\"5307161\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_783\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_155\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307161\" data-sigil=\"has-tooltip\" data-meta=\"0_782\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 5:19 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 17:19:24 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_780\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_781\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_156\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI detected an issue in the etcd pods:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e2019-07-04 17:16:06.697591 I | embed: rejected connection from "172.16.0.104:32810" (error "tls: failed to verify client's certificate: x509: certificate signed by unknown authority (possibly because of \\"crypto\/rsa: verification error\\" while trying to verify candidate authority certificate \\"etcd-ca\\")", ServerName "")\n2019-07-04 17:16:06.719517 I | embed: rejected connection from "172.16.0.104:32812" (error "tls: failed to verify client's certificate: x509: certificate signed by unknown authority (possibly because of \\"crypto\/rsa: verification error\\" while trying to verify candidate authority certificate \\"etcd-ca\\")", ServerName "")\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eAfter a while, etcd pods (and the api-server ones) start to crash and eventually the cluster dies. I wonder if we are overwriting something with puppet.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_793\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5307401\" id=\"5307401\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_792\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_157\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307401\" data-sigil=\"has-tooltip\" data-meta=\"0_791\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 9:12 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 21:12:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_789\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_790\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_158\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIs that happening in the one deployed with puppet distributed certs or using CLI only?  If using CLI only, it would be a bug in kubeadm, I think.  I wasn't seeing that with the puppet distributed (which is basically manual distribution)--at least as long as I watched it for, it may have snuck up later.\u003c\/p\u003e\n\n\u003cp\u003eOr...could it possibly be overwriting\/generating some certs when it does \u003ctt class=\"remarkup-monospaced\"\u003esudo kubeadm init phase upload-certs --upload-certs\u003c\/tt\u003e?  Like using the kubeapi CA cert still, but regenerating the etcd ones, which would cause that?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_802\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5307442\" id=\"5307442\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_801\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_159\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307442\" data-sigil=\"has-tooltip\" data-meta=\"0_800\"\u003e\u003cspan class=\"screen-only\"\u003eJul 4 2019, 9:30 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-04 21:30:27 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_798\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_799\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_160\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eMy thinking is that maybe it all works except when we do the upload-certs later to try to rebuild?  This would suggest we still need a manual cert copy for a rebuild, which isn't the end of the world.  Just more docs and\/or scripts (or even possibly adding the certs to labs\/private later on for that, which would work fine as well--and the way I did it in puppet, isn't used without a hiera trigger).  I don't know for sure without some logs or testing, though, obviously.  Just a guess.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_811\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5307979\" id=\"5307979\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_810\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_161\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5307979\" data-sigil=\"has-tooltip\" data-meta=\"0_809\"\u003e\u003cspan class=\"screen-only\"\u003eJul 5 2019, 9:58 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-05 09:58:58 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_807\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_808\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_162\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI think I know what is going on.\u003c\/p\u003e\n\n\u003cp\u003eIn my tests I did something like this:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ebootstrap the cluster\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eadd new members using the --upload-certs trick (testing a later upload)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003efor further investigation, then I deleted a master (kubeadm reset -f + \u003ctt class=\"remarkup-monospaced\"\u003ekubectl delete node <whatever\u003c\/tt\u003e)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003etry join again a control plane node using the --upload-certs trick.\u003c\/li\u003e\n\u003c\/ol\u003e\n\n\u003cp\u003eI think the problem here is that in the step 3, the etcd is not properly cleaned up by kubeadm.\u003cbr \/\u003e\nYou can see logs messages like the following:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e2019-07-05 09:48:51.686278 W | rafthttp: health check for peer dc517b72d6f67e5c could not connect: dial tcp 172.16.0.175:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")\n2019-07-05 09:48:51.719150 W | rafthttp: health check for peer dc517b72d6f67e5c could not connect: dial tcp 172.16.0.175:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe etcd cluster is still expecting to contact the deleted node! This means the etcd cluster is unhealthy, and any subsequent join operation for the control plane fails. This seems to me like a bug in kubeadm.\u003c\/p\u003e\n\n\u003cp\u003eSo, I see here 2 possible bugs:\u003c\/p\u003e\n\n\u003col class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ethe issue with etcd-ca and \u003ctt class=\"remarkup-monospaced\"\u003eSeverName ""\u003c\/tt\u003e, which I'm not sure yet what means.\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eetcd member not properly removed when reseting\/deleting a control plane node, preventing any further join operation for masters.\u003c\/li\u003e\n\u003c\/ol\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_820\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5308301\" id=\"5308301\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_819\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_163\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5308301\" data-sigil=\"has-tooltip\" data-meta=\"0_818\"\u003e\u003cspan class=\"screen-only\"\u003eJul 5 2019, 11:53 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-05 11:53:16 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_816\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_817\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_164\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI don't manage to do the join step for masters nodes using a config file. I can only successfully do it by using the cmdline generated by the first \u003ctt class=\"remarkup-monospaced\"\u003ekubadm init\u003c\/tt\u003e run.\u003c\/p\u003e\n\n\u003cp\u003eI've been trying with this config file (and some additional random modifications), which is a variant of the one installed by puppet:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eapiVersion: kubeadm.k8s.io\/v1beta2\nkind: JoinConfiguration\ndiscovery:\n  bootstrapToken:\n    apiServerEndpoint: toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443\n    token: "m7uakr.ern5lmlpv7gnkacw"\n    unsafeSkipCAVerification: true\n  timeout: 5m0s\n---\napiVersion: kubeadm.k8s.io\/v1beta2\nkind: JoinControlPlane\nlocalAPIEndpoint:\n  bindPort: 6443\nCertificateKey: "test"\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eSummary of the current issues I see\/have in the bootstrapping\/lifecycle workflow:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003ewe can only add a master if using the auto-generated cmdline for \u003ctt class=\"remarkup-monospaced\"\u003ekubeadm join\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ewe can not delete a master node, since that will leave etcd in an inconsistent state (even when explicitly running \u003ctt class=\"remarkup-monospaced\"\u003ekubeadm reset -f phase remove-etcd-member\u003c\/tt\u003e)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_829\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5308468\" id=\"5308468\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_828\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_165\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5308468\" data-sigil=\"has-tooltip\" data-meta=\"0_827\"\u003e\u003cspan class=\"screen-only\"\u003eJul 5 2019, 1:09 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-05 13:09:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_825\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_826\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_166\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI was able to re-join a master node to the cluster after a reset if:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003eyou don't delete \/var\/lib\/etcd after the reset\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003eyou run kubeadm join skipping a bunch of steps: \u003ctt class=\"remarkup-monospaced\"\u003ekubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443 --token m7uakr.ern5lmlpv7gnkacw     --discovery-token-ca-cert-hash sha256:whatever --experimental-control-plane --certificate-key whatever --ignore-preflight-errors=DirAvailable--var-lib-etcd --skip-phases=check-etcd,control-plane-join\/etcd\u003c\/tt\u003e\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ekubeadm will be unable to create the etcd pod, so you need to copy \u003ctt class=\"remarkup-monospaced\"\u003e\/etc\/kubernetes\/manifests\/etcd.yaml\u003c\/tt\u003e from other master node, update the IP addresses and then:\u003c\/li\u003e\n\u003c\/ul\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eroot@toolsbeta-test-k8s-master-3:~#\u003c\/span\u003e kubectl apply -f \/etc\/kubernetes\/manifests\/etcd.yaml \n\u003cspan class=\"go\"\u003epod\/etcd created\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_839\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5309280\" id=\"5309280\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_838\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_167\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003eEdited\u003cspan class=\"visual-only\" aria-hidden=\"true\"\u003e \u00b7 \u003c\/span\u003e\u003ca href=\"#5309280\" data-sigil=\"has-tooltip\" data-meta=\"0_837\"\u003e\u003cspan class=\"screen-only\"\u003eJul 5 2019, 4:10 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-05 16:10:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_835\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_836\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_168\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eYeah, it doesn't seem possible to join a master via the config file (on your earlier comment).  In at least one bug (\u003ca href=\"https:\/\/github.com\/kubernetes\/kubeadm\/issues\/1485\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/kubernetes\/kubeadm\/issues\/1485\u003c\/a\u003e), the developers stated that this is "by design" that --control-plane is only available for the CLI.  The only use in having the join config on a control plane node seems to be if we want to spin up later without the ca verification option.\u003c\/p\u003e\n\n\u003cp\u003eOverall, it seems like rebuilding a master node will not be fun and requires that we document all of this very well.  Thank you for figuring it out with the CLI.  It seems almost like it's actually smoother the way I'd done it using puppet?  Manually copying the certs either with puppet or CLI, the cluster seems to join up fast...but etcd will still have a problem if the node is replacing an old one, like you said.\u003c\/p\u003e\n\n\u003cp\u003eEtcd will not forget a node unless you force it to.  I don't think kubeadm has a direct interface into that (yet), so we'd have to use etcdctl \u003ca href=\"https:\/\/docs.okd.io\/latest\/admin_guide\/assembly_restore-etcd-quorum.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/docs.okd.io\/latest\/admin_guide\/assembly_restore-etcd-quorum.html\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eMaybe we need to make sure we know how to communicate with the containerized etcd.  It's probably no different as long as we have etcdctl installed somewhere.  If installed via package, it might try to start etcd on the node?  I don't know yet.  Have to look into that :-\/\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_848\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/s3vemh543felho7lpeeo\/PHID-FILE-fk66x77654xawrhes4qw\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Xinbenlv\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5316056\" id=\"5316056\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_847\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Xinbenlv\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_169\"\u003eXinbenlv\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5316056\" data-sigil=\"has-tooltip\" data-meta=\"0_846\"\u003e\u003cspan class=\"screen-only\"\u003eJul 9 2019, 5:46 AM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-09 05:46:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_844\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_845\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_170\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eSide comment: is there a way for developers (who are not WMF staff) to use them in beta?\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_857\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5317257\" id=\"5317257\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_856\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_171\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5317257\" data-sigil=\"has-tooltip\" data-meta=\"0_855\"\u003e\u003cspan class=\"screen-only\"\u003eJul 9 2019, 1:54 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-09 13:54:30 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_853\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_854\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_172\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T215531#5316056\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_17\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531#5316056\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Xinbenlv\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_18\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Xinbenlv\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eSide comment: is there a way for developers (who are not WMF staff) to use them in beta?\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWe don't have an actual kubernetes cluster yet. We build and dump it several times a day. I would suggest you wait until we have something ready for the Toolforge service.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_866\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5317268\" id=\"5317268\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_865\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_173\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5317268\" data-sigil=\"has-tooltip\" data-meta=\"0_864\"\u003e\u003cspan class=\"screen-only\"\u003eJul 9 2019, 1:57 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-09 13:57:28 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_862\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_863\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_174\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cblockquote class=\"remarkup-reply-block\"\u003e\n\u003cdiv class=\"remarkup-reply-head\"\u003eIn \u003ca href=\"\/T215531#5309280\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_19\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531#5309280\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e, \u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_20\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e wrote:\u003c\/div\u003e\n\u003cdiv class=\"remarkup-reply-body\"\u003e\u003cp\u003eYeah, it doesn't seem possible to join a master via the config file (on your earlier comment).  In at least one bug (\u003ca href=\"https:\/\/github.com\/kubernetes\/kubeadm\/issues\/1485\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/github.com\/kubernetes\/kubeadm\/issues\/1485\u003c\/a\u003e), the developers stated that this is "by design" that --control-plane is only available for the CLI.  The only use in having the join config on a control plane node seems to be if we want to spin up later without the ca verification option.\u003c\/p\u003e\n\n\u003cp\u003eOverall, it seems like rebuilding a master node will not be fun and requires that we document all of this very well.  Thank you for figuring it out with the CLI.  It seems almost like it's actually smoother the way I'd done it using puppet?  Manually copying the certs either with puppet or CLI, the cluster seems to join up fast...but etcd will still have a problem if the node is replacing an old one, like you said.\u003c\/p\u003e\n\n\u003cp\u003eEtcd will not forget a node unless you force it to.  I don't think kubeadm has a direct interface into that (yet), so we'd have to use etcdctl \u003ca href=\"https:\/\/docs.okd.io\/latest\/admin_guide\/assembly_restore-etcd-quorum.html\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/docs.okd.io\/latest\/admin_guide\/assembly_restore-etcd-quorum.html\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp\u003eMaybe we need to make sure we know how to communicate with the containerized etcd.  It's probably no different as long as we have etcdctl installed somewhere.  If installed via package, it might try to start etcd on the node?  I don't know yet.  Have to look into that :-\/\u003c\/p\u003e\u003c\/div\u003e\n\u003c\/blockquote\u003e\n\n\u003cp\u003eWe don't have etcdctl anywhere. I wonder if we should try using the external etcd cluster approach before anything else.\u003cbr \/\u003e\nCopying certificates once generated by kubeadm is not a very elegant solution I think. I would rather use puppet certs instead, but we already know that could be complex as well.\u003c\/p\u003e\n\n\u003cp\u003e:-\/\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_875\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5332792\" id=\"5332792\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_874\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_175\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5332792\" data-sigil=\"has-tooltip\" data-meta=\"0_873\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 12:15 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 12:15:40 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_871\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_872\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_176\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-tag-view phui-tag-type-person \" data-sigil=\"hovercard\" data-meta=\"0_21\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-person\"\u003e@Bstorm\u003c\/span\u003e\u003c\/a\u003e and I just had a meeting. We decided the following:\u003c\/p\u003e\n\n\u003cul class=\"remarkup-list\"\u003e\n\u003cli class=\"remarkup-list-item\"\u003etry using an external etcd server again. This time, using Debian Buster, which contains etcd 3.2.26+dfsg-3 (higher than the required 3.2.18) (\u003ca href=\"https:\/\/tracker.debian.org\/pkg\/etcd\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/tracker.debian.org\/pkg\/etcd\u003c\/a\u003e)\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ewe will try using puppet certs for the etcd server, and keep k8s using its own CA\u003c\/li\u003e\n\u003cli class=\"remarkup-list-item\"\u003ewe will eventually try kubeadm in debian buster as well. Mind iptables changes in Debian buster (kube-proxy, docker, etc)\u003c\/li\u003e\n\u003c\/ul\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_884\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/jo674bbqnw6mbz2qrulb\/PHID-FILE-kixxwfgiokzatzyk5rra\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Stashbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5332817\" id=\"5332817\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_883\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Stashbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_177\"\u003eStashbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5332817\" data-sigil=\"has-tooltip\" data-meta=\"0_882\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 12:27 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 12:27:14 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_880\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_881\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_178\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003e\u003cspan class=\"remarkup-nav-sequence\"\u003e\u003ca href=\"https:\/\/tools.wmflabs.org\/sal\/log\/AWv1mesnOwpQ-3PkETuX\" class=\"phui-tag-view phui-tag-type-shade phui-tag-grey phui-tag-shade phui-tag-icon-view \" target=\"_blank\" rel=\"noreferrer\"\u003e\u003cspan class=\"phui-tag-core \"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-file\" data-meta=\"0_4\" aria-hidden=\"true\"\u003e\u003c\/span\u003eMentioned in SAL (#wikimedia-cloud)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e [2019-07-15T12:27:13Z] <arturo> create \u003ctt class=\"remarkup-monospaced\"\u003etoolsbeta-test-k8s-etcd-1\u003c\/tt\u003e VM \u003ca href=\"\/T215531\" class=\"phui-tag-view phui-tag-type-object \" data-sigil=\"hovercard\" data-meta=\"0_22\"\u003e\u003cspan class=\"phui-tag-core-closed\"\u003e\u003cspan class=\"phui-tag-core phui-tag-color-object\"\u003eT215531\u003c\/span\u003e\u003c\/span\u003e\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_893\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6u4yyshlluybwy7amo56\/PHID-FILE-rn4megbghtdjqmko4ole\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/aborrero\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5333564\" id=\"5333564\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_892\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/aborrero\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_179\"\u003eaborrero\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5333564\" data-sigil=\"has-tooltip\" data-meta=\"0_891\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 3:06 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 15:06:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_889\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_890\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_180\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe new etcd server is ready (apparently):\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"shell-session\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e\u003cspan\u003e\u003c\/span\u003e\u003cspan class=\"gp\"\u003eaborrero@toolsbeta-test-k8s-etcd-1:~ $\u003c\/span\u003e sudo etcdctl --endpoints https:\/\/toolsbeta-test-k8s-etcd-1.toolsbeta.eqiad.wmflabs:2379 cluster-health\n\u003cspan class=\"go\"\u003emember 67a7255628c1f89f is healthy: got healthy result from https:\/\/toolsbeta-test-k8s-etcd-1.toolsbeta.eqiad.wmflabs:2379\u003c\/span\u003e\n\u003cspan class=\"go\"\u003ecluster is healthy\u003c\/span\u003e\u003c\/pre\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_902\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5333793\" id=\"5333793\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_901\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_181\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5333793\" data-sigil=\"has-tooltip\" data-meta=\"0_900\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 4:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 16:05:02 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_898\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_899\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_182\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 523220 had a related patch set uploaded (by Arturo Borrero Gonzalez; owner: Arturo Borrero Gonzalez):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: kubeadm: now using external etcd servers\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/523220\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/523220\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_911\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5333987\" id=\"5333987\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_910\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_183\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5333987\" data-sigil=\"has-tooltip\" data-meta=\"0_909\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 4:59 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 16:59:42 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_907\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_908\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_184\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eJust a comment on the certs issue:\u003cbr \/\u003e\nWith an external etcd cluster, the external cluster is in control of the server certs.  If it requires the certs to be in our config, then this version of etcd does require authentication (or this version of k8s does), which is honestly the right thing to do.  I was figuring we could continue to use it without auth and use the puppet certs like we used to.\u003c\/p\u003e\n\n\u003cp\u003eThe only way to ensure the auth works like this without distributing certs to the etcd servers (which I'd rather do via puppet than one more manually copied cert) is if we use puppet to generate the cert for the shared client cert.  We'd just have to use the puppet CA and openssl to do that (storing the result in the local labs-private).  Then we shouldn't need to copy any certs EXCEPT the etcd client cert.  That we'd have to copy to all k8s masters.  If the current running k8s master authenticates in toolforge (and I highly doubt it does), it would be using it's puppet host cert as a client cert.  That won't work with multiple masters.\u003c\/p\u003e\n\n\u003cp\u003eThe only other way I see doing this is if we spin up a separate etcd cluster using kubeadm (which would be containerized) rather than using the packages. \u003ca href=\"https:\/\/kubernetes.io\/docs\/setup\/production-environment\/tools\/kubeadm\/setup-ha-etcd-with-kubeadm\/\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/kubernetes.io\/docs\/setup\/production-environment\/tools\/kubeadm\/setup-ha-etcd-with-kubeadm\/\u003c\/a\u003e  <-- mind you, the method is not much better than creating a client cert using the puppet CA.  That's a lot of steps and still requires copying certs.\u003c\/p\u003e\n\n\u003cp\u003eI'm guessing based on your patch that it doesn't work without specifying the etcd client cert.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_920\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5333998\" id=\"5333998\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_919\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_185\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5333998\" data-sigil=\"has-tooltip\" data-meta=\"0_918\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 5:05 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 17:05:18 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_916\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_917\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_186\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWait, does it work without the client cert matching on every node (reading it more)?  I was expecting it to add that client cert to a configuration map in kubernetes, and I was worried it wouldn't work if they were different.  Maybe it doesn't matter because it just stores the location and all of them are valid individually :) That'd be awesome!\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_929\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5334353\" id=\"5334353\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_928\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_187\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5334353\" data-sigil=\"has-tooltip\" data-meta=\"0_927\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 6:42 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 18:42:49 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_925\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_926\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_188\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI went ahead to try and answer my own questions and noticed a problem.  We have etcd cert authentication enabled, and that cert is the puppet cert of the etcd server we've spun up.\u003cbr \/\u003e\nfrom \/etc\/default\/etcd\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eETCD_PEER_TRUSTED_CA_FILE="\/etc\/etcd\/ssl\/ca.pem"\nETCD_PEER_CERT_FILE="\/etc\/etcd\/ssl\/toolsbeta-test-k8s-etcd-1.toolsbeta.eqiad.wmflabs.pem"\nETCD_PEER_KEY_FILE="\/etc\/etcd\/ssl\/toolsbeta-test-k8s-etcd-1.toolsbeta.eqiad.wmflabs.priv"\nETCD_PEER_CLIENT_CERT_AUTH=true\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThat won't work since the other peers will have a different cert, if we use puppet certs.  \u003cbr \/\u003e\nI also see this:\u003cbr \/\u003e\n\u003ctt class=\"remarkup-monospaced\"\u003eETCD_CLIENT_CERT_AUTH=false\u003c\/tt\u003e which means the cert mentioned in the kubeadm config doesn't do a thing.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_938\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5334365\" id=\"5334365\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_937\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_189\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5334365\" data-sigil=\"has-tooltip\" data-meta=\"0_936\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 6:44 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 18:44:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_934\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_935\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_190\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWait, no.  That'll work or the peer cert file as long as that ca.pem is the puppet one.  Never mind :-p\u003c\/p\u003e\n\n\u003cp\u003eI'm curious about testing functioning if it doesn't use the client cert from kubernetes, though.  Will poke at it a bit.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_947\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5334479\" id=\"5334479\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_946\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_191\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5334479\" data-sigil=\"has-tooltip\" data-meta=\"0_945\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 7:07 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 19:07:06 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_943\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_944\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_192\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eFinally figured out how to query this version of etcd: \u003ctt class=\"remarkup-monospaced\"\u003eETCDCTL_API=3 etcdctl --endpoints https:\/\/toolsbeta-test-k8s-etcd-1.toolsbeta.eqiad.wmflabs:2379 get \/ --prefix --keys-only\u003c\/tt\u003e\u003c\/p\u003e\n\n\u003cp\u003eAnyway, since etcd (in this state) isn't even checking the client cert, what I was curious about really doesn't matter, and I think the whole setup will scale fine.  Maybe we should test it, though? It'll break using etcdctl with explicit endpoints outside of localhost.  That said, I'm going to maybe put up a patch that allows localhost through the firewall.  *if* we do decide to validate client certs (which is best practice), then localhost is the only place where etcdctl will work (unless that needs a cert too...which it might with that in place?)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_956\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5334533\" id=\"5334533\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_955\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_193\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5334533\" data-sigil=\"has-tooltip\" data-meta=\"0_954\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 7:24 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 19:24:55 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_952\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_953\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_194\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChanged my mind on that last bit because you can specify certs with etcdctl :)  No need to skip ssl whether we enable the verification or not.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_965\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5334856\" id=\"5334856\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_964\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_195\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5334856\" data-sigil=\"has-tooltip\" data-meta=\"0_963\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 8:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 20:47:37 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_961\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_962\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_196\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 523220 \u003cstrong\u003emerged\u003c\/strong\u003e by Bstorm:\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: k8s: kubeadm: now using external etcd servers\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/523220\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/523220\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_974\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5334868\" id=\"5334868\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_973\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_197\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5334868\" data-sigil=\"has-tooltip\" data-meta=\"0_972\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 8:51 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 20:51:33 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_970\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_971\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_198\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eGoing to try to bootstrap the other two cluster nodes using the --upload-certs thing since I haven't tried it myself yet :)\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_983\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335032\" id=\"5335032\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_982\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_199\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335032\" data-sigil=\"has-tooltip\" data-meta=\"0_981\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 9:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 21:36:05 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_979\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_980\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_200\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eIt does *not* work as merged.\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003eroot@toolsbeta-test-k8s-master-2:~# kubeadm join toolsbeta-k8s-master.toolsbeta.wmflabs.org:6443  --token m7uakr.ern5lmlpv7gnkacw --control-plane --discovery-token-ca-cert-hash sha256:<hash> --certificate-key <key-from-upload-certs-command>\n[preflight] Running pre-flight checks\n[preflight] Reading configuration from the cluster...\n[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'\n[preflight] Running pre-flight checks before initializing the new control plane instance\n[preflight] Pulling images required for setting up a Kubernetes cluster\n[preflight] This might take a minute or two, depending on the speed of your internet connection\n[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'\n[download-certs] Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace\nerror execution phase control-plane-prepare\/download-certs: error downloading certs: the Secret does not include the required certificate or key - name: external-etcd.key, path: \/etc\/kubernetes\/pki\/puppet_toolsbeta-test-k8s-master-1.toolsbeta.eqiad.wmflabs.priv\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThe nodes would all need the same puppet cert.  Since the etcd cluster isn't checking client certs anyway, we should attempt to remove the cert portions of the config.  Lemme try that.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_992\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335033\" id=\"5335033\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_991\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_201\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335033\" data-sigil=\"has-tooltip\" data-meta=\"0_990\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 9:36 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 21:36:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_988\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_989\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_202\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eI added a command to quickly get the ca-cert-hash, btw, in the wiki page of notes.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1001\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335083\" id=\"5335083\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1000\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_203\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335083\" data-sigil=\"has-tooltip\" data-meta=\"0_999\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 9:50 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 21:50:43 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_997\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_998\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_204\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eWithout it I see this problem with calico:\u003c\/p\u003e\n\n\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# kubectl logs calico-node-qk4kf --namespace=kube-system \n2019-07-15 21:49:12.134 [INFO][9] startup.go 256: Early log level set to info\n2019-07-15 21:49:12.134 [INFO][9] startup.go 272: Using NODENAME environment for node name\n2019-07-15 21:49:12.134 [INFO][9] startup.go 284: Determined node name: toolsbeta-test-k8s-master-1\n2019-07-15 21:49:12.136 [INFO][9] k8s.go 228: Using Calico IPAM\n2019-07-15 21:49:12.136 [INFO][9] startup.go 316: Checking datastore connection\n2019-07-15 21:49:12.146 [WARNING][9] startup.go 328: Connection to the datastore is unauthorized\n2019-07-15 21:49:12.146 [WARNING][9] startup.go 1057: Terminating\nCalico node failed to start\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003eThat may be due to old config maps.  I might try re-initializing etcd.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1010\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335085\" id=\"5335085\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1009\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_205\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335085\" data-sigil=\"has-tooltip\" data-meta=\"0_1008\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 9:52 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 21:52:50 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1006\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1007\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_206\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eReset etcd with \u003ctt class=\"remarkup-monospaced\"\u003eETCDCTL_API=3 etcdctl --endpoints https:\/\/toolsbeta-test-k8s-etcd-1.toolsbeta.eqiad.wmflabs:2379 del "" --from-key=true\u003c\/tt\u003e as normal user.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1019\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335106\" id=\"5335106\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1018\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_207\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335106\" data-sigil=\"has-tooltip\" data-meta=\"0_1017\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 9:56 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 21:56:51 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1015\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1016\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_208\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eVictory!  Now I'll try to join another control plane node.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1028\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335114\" id=\"5335114\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1027\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_209\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335114\" data-sigil=\"has-tooltip\" data-meta=\"0_1026\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 10:01 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 22:01:59 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1024\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1025\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_210\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cdiv class=\"remarkup-code-block\" data-code-lang=\"text\" data-sigil=\"remarkup-code-block\"\u003e\u003cpre class=\"remarkup-code\"\u003e# kubectl get nodes\nNAME                          STATUS   ROLES    AGE     VERSION\ntoolsbeta-test-k8s-master-1   Ready    master   7m24s   v1.15.0\ntoolsbeta-test-k8s-master-2   Ready    master   3m46s   v1.15.0\ntoolsbeta-test-k8s-master-3   Ready    master   2m55s   v1.15.0\u003c\/pre\u003e\u003c\/div\u003e\n\n\u003cp\u003ePushing a patch.  This works great.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1037\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6vn6slgby7ia62ouikut\/PHID-FILE-ay56qvafgaxajuctgtw3\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/gerritbot\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335116\" id=\"5335116\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1036\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/gerritbot\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_211\"\u003egerritbot\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335116\" data-sigil=\"has-tooltip\" data-meta=\"0_1035\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 10:02 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 22:02:39 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1033\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1034\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_212\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eChange 523328 had a related patch set uploaded (by Bstorm; owner: Bstorm):\u003cbr \/\u003e\n[operations\/puppet@production] toolforge: kubeadm master nodes shouldn't use client certs for etcd\u003c\/p\u003e\n\n\u003cp\u003e\u003ca href=\"https:\/\/gerrit.wikimedia.org\/r\/523328\" class=\"remarkup-link\" target=\"_blank\" rel=\"noreferrer\"\u003ehttps:\/\/gerrit.wikimedia.org\/r\/523328\u003c\/a\u003e\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1046\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335247\" id=\"5335247\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1045\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_213\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335247\" data-sigil=\"has-tooltip\" data-meta=\"0_1044\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 10:45 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 22:45:26 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1042\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1043\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_214\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cp\u003eThe alternative is, obviously, to use a client cert that is held in common by all the nodes (with each of their names on it) and turn on client cert checking.  That cert can be made using the \u003ctt class=\"remarkup-monospaced\"\u003epuppet cert generate\u003c\/tt\u003e command with the --dns_alt_names option including the names of all three master nodes.  I tested that process in another project.  It's kind of weird (puts the resulting files in \/var\/lib\/puppet\/ssl\/server\/ where it keeps the original master certs made during bootstrap, but it didn't seem to break anything where I tested it).  I can't say I like it, but it might be good.  I mean, with access to the puppetmaster, one can also just use the openssl CLI to make and sign a cert for this that will be trusted by etcd.\u003c\/p\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-spacer\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-shell\" data-sigil=\"transaction anchor-container\" data-meta=\"0_1055\"\u003e\u003cdiv class=\"phui-timeline-event-view phui-timeline-major-event\"\u003e\u003cdiv class=\"phui-timeline-content\"\u003e\u003ca style=\"background-image: url(https:\/\/phab.wmfusercontent.org\/file\/data\/6joq4pwcvg4u43sjvknf\/PHID-FILE-h2b2vilwlub2bcfhkbnd\/profile)\" class=\"visual-only phui-timeline-image\" href=\"\/p\/Bstorm\/\" aria-hidden=\"true\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-wedge\" style=\"\"\u003e\u003c\/div\u003e\u003cdiv class=\"phui-timeline-group\"\u003e\u003cdiv class=\"phui-timeline-inner-content\"\u003e\u003ca name=\"5335262\" id=\"5335262\" class=\"phabricator-anchor-view\"\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-title phui-timeline-title-with-icon phui-timeline-title-with-menu\"\u003e\u003cspan class=\"phui-timeline-icon-fill\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-comment phui-timeline-icon\" data-meta=\"0_1054\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/span\u003e\u003ca href=\"\/p\/Bstorm\/\" class=\"phui-handle phui-link-person\" data-sigil=\"hovercard\" data-meta=\"0_215\"\u003eBstorm\u003c\/a\u003e added a comment.\u003cspan class=\"phui-timeline-extra\"\u003e\u003ca href=\"#5335262\" data-sigil=\"has-tooltip\" data-meta=\"0_1053\"\u003e\u003cspan class=\"screen-only\"\u003eJul 15 2019, 10:47 PM\u003c\/span\u003e\u003cspan class=\"print-only\" aria-hidden=\"true\"\u003e2019-07-15 22:47:41 (UTC+0)\u003c\/span\u003e\u003c\/a\u003e\u003c\/span\u003e\u003c\/div\u003e\u003ca href=\"#\" class=\"phui-timeline-menu\" aria-haspopup=\"true\" aria-expanded=\"false\" data-sigil=\"phui-dropdown-menu\" data-meta=\"0_1051\"\u003e\u003cspan class=\"aural-only\"\u003eComment Actions\u003c\/span\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-caret-down\" data-meta=\"0_1052\" aria-hidden=\"true\"\u003e\u003c\/span\u003e\u003c\/a\u003e\u003cdiv class=\"phui-timeline-core-content\"\u003e\u003cspan class=\"transaction-comment\" data-sigil=\"transaction-comment\" data-meta=\"0_216\"\u003e\u003cdiv class=\"phabricator-remarkup\"\u003e\u003cdiv class=\"remarkup-note\"\u003e\u003cspan class=\"remarkup-note-word\"\u003eNOTE:\u003c\/span\u003e puppet is disabled on master-1 where I was livehacking--for when you try things in your morning.  Feel free to re-enable and mess with things, of course.  I didn't un-hack anything on the puppetmaster itself\u003c\/div\u003e\u003c\/div\u003e\u003c\/span\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e\u003c\/div\u003e"},"javelin_metadata":[{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},[],[],[],[],{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-zzrippyp4ljdd6gvhv4z"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-2gyt74kbwhlivylgixsc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-USER-dw4wihyk4lbzl3ephpmn","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr","contextPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-vzliwhqljwra2fkn5a76"}},{"phid":"PHID-XACT-TASK-prtfzj5ela3pfei"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-d2kjuc3t7stcas7"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-tuilekxlt3zmhca"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-5x6gr6ixvhtlarl"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-7jgyx6j32bkvzdr"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-iwjdasnw5s7hlaq"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-pcqkpukocy52yr6"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-ewnqmulon7ze3cu"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-z6icufyeygysg3p"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-ecfqtvgddxilty5"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-pghmtp4ngkn6d3j"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-hq6pk2bui3nbnpd"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-uaa4vaw57pxtkoj"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-jg452aqyn5n262m"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-x74cmkusagaxswq"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-5zepay4ifr5pl4q"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-rsdarrc43bu476d"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-t4njxvgvlkqgv2g"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-eaqqu4dbacix72o"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-kywhx6u447qsyji"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-as5znvaqzxlvhir"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-fdowjs4od5xvepl"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-cqlerj4crk6wvvf"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-izjpgwfranpntcr"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-nwf73d2747npfgy"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-mshrg47uioxsfhr"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-ui7e4r7cm2nqyjz"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-lqfg2wcogzfysxc"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-m7d6mbl3ehst44e"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-56tbwvk2b5gvfak"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-mi3giyhclncwv3u"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-im7ga6osilldofn"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-avyanm24lvlzuau"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-bbovqg5yhh4jaia"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-ubjgdxkmnuz7dd6"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-kbnoygk6npyrbwn"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-xsujo6a2lye5gvl"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-aatqapdt7ow4fja"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-ue7jnjpzq3rodsv"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-psv246a374guafe"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-ki4hscyy3yrgdza"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-3ayuwvdsuq6khcg"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-ekdblxyhnpau2nf"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-g6orb5vxu3b7bvr"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-535zozjrmti4fzd"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-v47boaiat3wkrfu"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-73dtecrirwsst6i"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-7mntc3pkihuabx5"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-b3exf5rqzvuvlqf"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-7ewqso37qffhynb"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-m2kyhqymhaa5qck"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-kjvujdvrvutk7ou"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-pcljcvgstht4udp"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-kwqfcxzt7v5laz7"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-2u6xm43uicethsk"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-tgavhc3hjo2lvyl"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-t32lqgoohh5hxbb"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-xrjtlc4mlheexmi"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-nben6zz4ho35e7j"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yabsmfq5qebjqm5te7ts"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-yazt5jyba6s43e64umxc"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ky52iujeyw6x5vu22r7a"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-xxanrjiws3nin53"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-rqh5opkhafhbfuyujsgs"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-ibnny2mpq3tjmih643rb"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"hovercardSpec":{"objectPHID":"PHID-TASK-lsuvd7mwodsigiz7yg2f"}},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-gtqioiu7tnwo7ip"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-37vhsiw262sx4s3"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-mo6wnloagu44f74"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-kvksbn7g4hq25ed"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-ss3csl54agzre6a"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-5zvj62iq42lqyjx"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-3r27zov2euduzmd"},{"hovercardSpec":{"objectPHID":"PHID-USER-dw4wihyk4lbzl3ephpmn"}},{"phid":"PHID-XACT-TASK-ihtuk3g4ymyvwr5"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-jkowvozclaedds4"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-ah4f7eaqxubsvqd"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-jz3cqifgalxkzv2"},{"hovercardSpec":{"objectPHID":"PHID-USER-j4uyesgqhubl2dywl4xd"}},{"phid":"PHID-XACT-TASK-rqh3hjiav6yopjr"},{"hovercardSpec":{"objectPHID":"PHID-USER-yue54ihblxe4zj3mpohv"}},{"phid":"PHID-XACT-TASK-xopah5clof3pzhx"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-4q4kk5an6gfu3ok"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-7a2ol45yum73qqa"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-dqbozzyycbssimc"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-uzfiwnjoiilx5qq"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-5hkkyxbkb4sbsia"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-syaopppyszetno6"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-kj2qwslt4is5h6v"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-glz4hmvqhzmtbai"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-j527rjqddrpphql"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-s2ls26vdzybja75"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-rxaliuwduwmnfr5"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-u7a6nnnertcja3v"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-rxpnnpmfdpzgqx4"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-we7glrklo4z5gzv"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-lmnpubqmq725i3m"},{"hovercardSpec":{"objectPHID":"PHID-USER-idceizaw6elwiwm5xshb"}},{"phid":"PHID-XACT-TASK-gu5iox6bfrl74t3"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-yosruzjhjjztszz"},{"hovercardSpec":{"objectPHID":"PHID-USER-etxrnkbidavgyjmq6odr"}},{"phid":"PHID-XACT-TASK-dmx3a52gwdnvp5j"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-prtfzj5ela3pfei\/","ref":"T215531#5301723"},[],{"anchor":"5301723"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_1\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_217\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_218\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_3\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-prtfzj5ela3pfei\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_219\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_220\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-prtfzj5ela3pfei","anchor":"5301723"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-d2kjuc3t7stcas7\/","ref":"T215531#5301739"},[],{"anchor":"5301739"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_5\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_226\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_227\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_7\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-d2kjuc3t7stcas7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_228\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_229\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-d2kjuc3t7stcas7","anchor":"5301739"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tuilekxlt3zmhca\/","ref":"T215531#5301811"},[],{"anchor":"5301811"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_9\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_235\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_236\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_11\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tuilekxlt3zmhca\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_237\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_238\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-tuilekxlt3zmhca","anchor":"5301811"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5x6gr6ixvhtlarl\/","ref":"T215531#5301901"},[],{"anchor":"5301901"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_13\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_244\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_245\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_15\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5x6gr6ixvhtlarl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_246\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_247\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5x6gr6ixvhtlarl","anchor":"5301901"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7jgyx6j32bkvzdr\/","ref":"T215531#5301932"},[],{"anchor":"5301932"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_17\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_253\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_254\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_19\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7jgyx6j32bkvzdr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_255\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_256\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7jgyx6j32bkvzdr","anchor":"5301932"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-iwjdasnw5s7hlaq\/","ref":"T215531#5301942"},[],{"anchor":"5301942"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_21\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_262\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_263\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_23\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-iwjdasnw5s7hlaq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_264\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_265\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-iwjdasnw5s7hlaq","anchor":"5301942"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pcqkpukocy52yr6\/","ref":"T215531#5301987"},[],{"anchor":"5301987"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_25\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_271\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_272\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_27\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pcqkpukocy52yr6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_273\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_274\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-pcqkpukocy52yr6","anchor":"5301987"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ewnqmulon7ze3cu\/","ref":"T215531#5301990"},[],{"anchor":"5301990"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_29\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_280\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_281\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_31\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ewnqmulon7ze3cu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_282\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_283\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ewnqmulon7ze3cu","anchor":"5301990"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-z6icufyeygysg3p\/","ref":"T215531#5302016"},[],{"anchor":"5302016"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_33\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_289\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_290\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_35\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-z6icufyeygysg3p\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_291\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_292\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-z6icufyeygysg3p","anchor":"5302016"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ecfqtvgddxilty5\/","ref":"T215531#5302020"},[],{"anchor":"5302020"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_37\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_298\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_299\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_39\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ecfqtvgddxilty5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_300\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_301\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ecfqtvgddxilty5","anchor":"5302020"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pghmtp4ngkn6d3j\/","ref":"T215531#5302067"},[],{"anchor":"5302067"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_41\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_307\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_308\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_43\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pghmtp4ngkn6d3j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_309\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_310\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pghmtp4ngkn6d3j","anchor":"5302067"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-hq6pk2bui3nbnpd\/","ref":"T215531#5302118"},[],{"anchor":"5302118"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_45\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_316\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_317\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_47\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-hq6pk2bui3nbnpd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_318\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_319\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-hq6pk2bui3nbnpd","anchor":"5302118"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uaa4vaw57pxtkoj\/","ref":"T215531#5302129"},[],{"anchor":"5302129"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_49\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_325\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_326\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_51\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uaa4vaw57pxtkoj\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_327\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_328\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uaa4vaw57pxtkoj","anchor":"5302129"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jg452aqyn5n262m\/","ref":"T215531#5302182"},[],{"anchor":"5302182"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_53\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_334\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_335\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_55\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jg452aqyn5n262m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_336\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_337\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jg452aqyn5n262m","anchor":"5302182"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-x74cmkusagaxswq\/","ref":"T215531#5302894"},[],{"anchor":"5302894"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_57\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_343\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_344\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_59\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-x74cmkusagaxswq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_345\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_346\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-x74cmkusagaxswq","anchor":"5302894"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5zepay4ifr5pl4q\/","ref":"T215531#5302900"},[],{"anchor":"5302900"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_61\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_352\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_353\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_63\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5zepay4ifr5pl4q\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_354\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_355\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-5zepay4ifr5pl4q","anchor":"5302900"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rsdarrc43bu476d\/","ref":"T215531#5302911"},[],{"anchor":"5302911"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_65\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_361\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_362\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_67\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rsdarrc43bu476d\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_363\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_364\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rsdarrc43bu476d","anchor":"5302911"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-t4njxvgvlkqgv2g\/","ref":"T215531#5302925"},[],{"anchor":"5302925"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_69\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_370\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_371\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_71\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t4njxvgvlkqgv2g\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_372\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_373\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-t4njxvgvlkqgv2g","anchor":"5302925"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-eaqqu4dbacix72o\/","ref":"T215531#5302939"},[],{"anchor":"5302939"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_73\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_379\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_380\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_75\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-eaqqu4dbacix72o\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_381\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_382\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-eaqqu4dbacix72o","anchor":"5302939"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kywhx6u447qsyji\/","ref":"T215531#5302972"},[],{"anchor":"5302972"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_77\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_388\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_389\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_79\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kywhx6u447qsyji\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_390\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_391\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-kywhx6u447qsyji","anchor":"5302972"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-as5znvaqzxlvhir\/","ref":"T215531#5302976"},[],{"anchor":"5302976"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_81\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_397\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_398\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_83\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-as5znvaqzxlvhir\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_399\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_400\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-as5znvaqzxlvhir","anchor":"5302976"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-fdowjs4od5xvepl\/","ref":"T215531#5303023"},[],{"anchor":"5303023"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_85\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_406\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_407\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_87\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-fdowjs4od5xvepl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_408\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_409\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-fdowjs4od5xvepl","anchor":"5303023"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-cqlerj4crk6wvvf\/","ref":"T215531#5303025"},[],{"anchor":"5303025"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_89\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_415\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_416\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_91\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-cqlerj4crk6wvvf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_417\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_418\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-cqlerj4crk6wvvf","anchor":"5303025"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-izjpgwfranpntcr\/","ref":"T215531#5303118"},[],{"anchor":"5303118"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_93\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_424\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_425\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_95\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-izjpgwfranpntcr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_426\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_427\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-izjpgwfranpntcr","anchor":"5303118"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nwf73d2747npfgy\/","ref":"T215531#5303121"},[],{"anchor":"5303121"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_97\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_433\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_434\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_99\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nwf73d2747npfgy\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_435\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_436\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-nwf73d2747npfgy","anchor":"5303121"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mshrg47uioxsfhr\/","ref":"T215531#5303182"},[],{"anchor":"5303182"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_101\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_442\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_443\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_103\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mshrg47uioxsfhr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_444\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_445\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mshrg47uioxsfhr","anchor":"5303182"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ui7e4r7cm2nqyjz\/","ref":"T215531#5303184"},[],{"anchor":"5303184"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_105\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_451\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_452\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_107\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ui7e4r7cm2nqyjz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_453\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_454\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-ui7e4r7cm2nqyjz","anchor":"5303184"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lqfg2wcogzfysxc\/","ref":"T215531#5303334"},[],{"anchor":"5303334"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_109\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_460\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_461\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_111\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lqfg2wcogzfysxc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_462\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_463\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lqfg2wcogzfysxc","anchor":"5303334"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-m7d6mbl3ehst44e\/","ref":"T215531#5303556"},[],{"anchor":"5303556"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_113\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_469\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_470\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_115\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-m7d6mbl3ehst44e\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_471\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_472\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-m7d6mbl3ehst44e","anchor":"5303556"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-56tbwvk2b5gvfak\/","ref":"T215531#5303645"},[],{"anchor":"5303645"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_117\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_478\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_479\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_119\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-56tbwvk2b5gvfak\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_480\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_481\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-56tbwvk2b5gvfak","anchor":"5303645"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mi3giyhclncwv3u\/","ref":"T215531#5303646"},[],{"anchor":"5303646"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_121\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_487\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_488\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_123\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mi3giyhclncwv3u\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_489\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_490\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mi3giyhclncwv3u","anchor":"5303646"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-im7ga6osilldofn\/","ref":"T215531#5303651"},[],{"anchor":"5303651"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_125\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_496\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_497\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_127\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-im7ga6osilldofn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_498\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_499\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-im7ga6osilldofn","anchor":"5303651"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-avyanm24lvlzuau\/","ref":"T215531#5303667"},[],{"anchor":"5303667"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_129\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_505\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_506\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_131\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-avyanm24lvlzuau\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_507\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_508\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-avyanm24lvlzuau","anchor":"5303667"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-bbovqg5yhh4jaia\/","ref":"T215531#5303680"},[],{"anchor":"5303680"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_133\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_514\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_515\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_135\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-bbovqg5yhh4jaia\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_516\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_517\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bbovqg5yhh4jaia","anchor":"5303680"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ubjgdxkmnuz7dd6\/","ref":"T215531#5303687"},[],{"anchor":"5303687"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_137\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_523\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_524\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_139\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ubjgdxkmnuz7dd6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_525\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_526\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ubjgdxkmnuz7dd6","anchor":"5303687"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kbnoygk6npyrbwn\/","ref":"T215531#5303690"},[],{"anchor":"5303690"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_141\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_532\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_533\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_143\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kbnoygk6npyrbwn\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_534\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_535\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kbnoygk6npyrbwn","anchor":"5303690"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xsujo6a2lye5gvl\/","ref":"T215531#5303706"},[],{"anchor":"5303706"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_145\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_541\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_542\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_147\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xsujo6a2lye5gvl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_543\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_544\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xsujo6a2lye5gvl","anchor":"5303706"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-aatqapdt7ow4fja\/","ref":"T215531#5303736"},[],{"anchor":"5303736"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_149\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_550\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_551\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_151\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-aatqapdt7ow4fja\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_552\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_553\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-aatqapdt7ow4fja","anchor":"5303736"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ue7jnjpzq3rodsv\/","ref":"T215531#5303750"},[],{"anchor":"5303750"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_153\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_559\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_560\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_155\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ue7jnjpzq3rodsv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_561\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_562\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ue7jnjpzq3rodsv","anchor":"5303750"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-psv246a374guafe\/","ref":"T215531#5303756"},[],{"anchor":"5303756"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_157\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_568\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_569\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_159\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-psv246a374guafe\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_570\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_571\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-psv246a374guafe","anchor":"5303756"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ki4hscyy3yrgdza\/","ref":"T215531#5304209"},[],{"anchor":"5304209"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_161\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_577\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_578\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_163\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ki4hscyy3yrgdza\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_579\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_580\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ki4hscyy3yrgdza","anchor":"5304209"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3ayuwvdsuq6khcg\/","ref":"T215531#5304576"},[],{"anchor":"5304576"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_165\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_586\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_587\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_167\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3ayuwvdsuq6khcg\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_588\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_589\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-3ayuwvdsuq6khcg","anchor":"5304576"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ekdblxyhnpau2nf\/","ref":"T215531#5304619"},[],{"anchor":"5304619"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_169\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_595\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_596\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_171\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ekdblxyhnpau2nf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_597\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_598\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ekdblxyhnpau2nf","anchor":"5304619"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-g6orb5vxu3b7bvr\/","ref":"T215531#5304659"},[],{"anchor":"5304659"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_173\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_604\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_605\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_175\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-g6orb5vxu3b7bvr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_606\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_607\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-g6orb5vxu3b7bvr","anchor":"5304659"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-535zozjrmti4fzd\/","ref":"T215531#5304751"},[],{"anchor":"5304751"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_177\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_613\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_614\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_179\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-535zozjrmti4fzd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_615\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_616\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-535zozjrmti4fzd","anchor":"5304751"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-v47boaiat3wkrfu\/","ref":"T215531#5304770"},[],{"anchor":"5304770"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_181\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_622\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_623\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_183\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-v47boaiat3wkrfu\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_624\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_625\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-v47boaiat3wkrfu","anchor":"5304770"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-73dtecrirwsst6i\/","ref":"T215531#5304776"},[],{"anchor":"5304776"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_185\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_631\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_632\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_187\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-73dtecrirwsst6i\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_633\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_634\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-73dtecrirwsst6i","anchor":"5304776"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7mntc3pkihuabx5\/","ref":"T215531#5304780"},[],{"anchor":"5304780"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_189\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_640\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_641\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_191\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7mntc3pkihuabx5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_642\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_643\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-7mntc3pkihuabx5","anchor":"5304780"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-b3exf5rqzvuvlqf\/","ref":"T215531#5304868"},[],{"anchor":"5304868"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_193\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_649\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_650\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_195\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-b3exf5rqzvuvlqf\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_651\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_652\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-b3exf5rqzvuvlqf","anchor":"5304868"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7ewqso37qffhynb\/","ref":"T215531#5304901"},[],{"anchor":"5304901"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_197\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_658\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_659\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_199\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7ewqso37qffhynb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_660\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_661\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7ewqso37qffhynb","anchor":"5304901"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-m2kyhqymhaa5qck\/","ref":"T215531#5304907"},[],{"anchor":"5304907"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_201\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_667\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_668\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_203\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-m2kyhqymhaa5qck\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_669\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_670\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-m2kyhqymhaa5qck","anchor":"5304907"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kjvujdvrvutk7ou\/","ref":"T215531#5304916"},[],{"anchor":"5304916"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_205\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_676\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_677\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_207\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kjvujdvrvutk7ou\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_678\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_679\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kjvujdvrvutk7ou","anchor":"5304916"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-pcljcvgstht4udp\/","ref":"T215531#5304978"},[],{"anchor":"5304978"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_209\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_685\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_686\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_211\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-pcljcvgstht4udp\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_687\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_688\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-pcljcvgstht4udp","anchor":"5304978"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kwqfcxzt7v5laz7\/","ref":"T215531#5304981"},[],{"anchor":"5304981"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_213\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_694\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_695\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_215\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kwqfcxzt7v5laz7\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_696\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_697\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kwqfcxzt7v5laz7","anchor":"5304981"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-2u6xm43uicethsk\/","ref":"T215531#5306564"},[],{"anchor":"5306564"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_217\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_703\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_704\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_219\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-2u6xm43uicethsk\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_705\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_706\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-2u6xm43uicethsk","anchor":"5306564"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-tgavhc3hjo2lvyl\/","ref":"T215531#5306644"},[],{"anchor":"5306644"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_221\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_712\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_713\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_223\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-tgavhc3hjo2lvyl\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_714\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_715\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-tgavhc3hjo2lvyl","anchor":"5306644"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-t32lqgoohh5hxbb\/","ref":"T215531#5306869"},[],{"anchor":"5306869"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_225\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_721\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_722\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_227\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-t32lqgoohh5hxbb\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_723\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_724\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_229\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-t32lqgoohh5hxbb\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_725\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-t32lqgoohh5hxbb","anchor":"5306869"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xrjtlc4mlheexmi\/","ref":"T215531#5306882"},[],{"anchor":"5306882"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_231\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_731\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_732\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_233\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xrjtlc4mlheexmi\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_733\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_734\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xrjtlc4mlheexmi","anchor":"5306882"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-nben6zz4ho35e7j\/","ref":"T215531#5306950"},[],{"anchor":"5306950"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_235\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_740\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_741\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_237\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-nben6zz4ho35e7j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_742\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_743\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-nben6zz4ho35e7j","anchor":"5306950"},{"tip":"Via Web"},[],[],[],{"phid":"PHID-XACT-TASK-grgrphrgxurbeyc","anchor":"5307014"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-bclhmx4jvwjkng3","anchor":"5307052"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xxanrjiws3nin53\/","ref":"T215531#5307104"},[],{"anchor":"5307104"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_239\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_757\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_758\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_241\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xxanrjiws3nin53\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_759\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_760\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_243\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-xxanrjiws3nin53\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_761\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xxanrjiws3nin53","anchor":"5307104"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-6znbhkpqgs53iwf","anchor":"5307110"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-77hrhubgkmjf62e","anchor":"5307120"},{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-4srxwhmwvwxqfcd","anchor":"5307145"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gtqioiu7tnwo7ip\/","ref":"T215531#5307161"},[],{"anchor":"5307161"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_245\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_776\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_777\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_247\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gtqioiu7tnwo7ip\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_778\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_779\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-gtqioiu7tnwo7ip","anchor":"5307161"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-37vhsiw262sx4s3\/","ref":"T215531#5307401"},[],{"anchor":"5307401"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_249\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_785\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_786\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_251\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-37vhsiw262sx4s3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_787\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_788\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-37vhsiw262sx4s3","anchor":"5307401"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-mo6wnloagu44f74\/","ref":"T215531#5307442"},[],{"anchor":"5307442"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_253\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_794\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_795\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_255\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-mo6wnloagu44f74\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_796\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_797\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-mo6wnloagu44f74","anchor":"5307442"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kvksbn7g4hq25ed\/","ref":"T215531#5307979"},[],{"anchor":"5307979"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_257\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_803\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_804\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_259\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kvksbn7g4hq25ed\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_805\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_806\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kvksbn7g4hq25ed","anchor":"5307979"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ss3csl54agzre6a\/","ref":"T215531#5308301"},[],{"anchor":"5308301"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_261\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_812\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_813\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_263\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ss3csl54agzre6a\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_814\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_815\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ss3csl54agzre6a","anchor":"5308301"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5zvj62iq42lqyjx\/","ref":"T215531#5308468"},[],{"anchor":"5308468"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_265\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_821\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_822\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_267\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5zvj62iq42lqyjx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_823\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_824\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5zvj62iq42lqyjx","anchor":"5308468"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-3r27zov2euduzmd\/","ref":"T215531#5309280"},[],{"anchor":"5309280"},[],[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_269\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_830\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_831\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_271\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-3r27zov2euduzmd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_832\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_833\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_273\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/history\/PHID-XACT-TASK-3r27zov2euduzmd\/\" class=\"phabricator-action-view-item\" data-sigil=\"workflow\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-list phabricator-action-view-icon\" data-meta=\"0_834\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Edit History\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-3r27zov2euduzmd","anchor":"5309280"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ihtuk3g4ymyvwr5\/","ref":"T215531#5316056"},[],{"anchor":"5316056"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_275\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_840\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_841\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_277\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ihtuk3g4ymyvwr5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_842\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_843\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ihtuk3g4ymyvwr5","anchor":"5316056"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jkowvozclaedds4\/","ref":"T215531#5317257"},[],{"anchor":"5317257"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_279\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_849\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_850\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_281\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jkowvozclaedds4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_851\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_852\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jkowvozclaedds4","anchor":"5317257"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-ah4f7eaqxubsvqd\/","ref":"T215531#5317268"},[],{"anchor":"5317268"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_283\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_858\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_859\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_285\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-ah4f7eaqxubsvqd\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_860\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_861\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-ah4f7eaqxubsvqd","anchor":"5317268"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-jz3cqifgalxkzv2\/","ref":"T215531#5332792"},[],{"anchor":"5332792"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_287\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_867\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_868\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_289\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-jz3cqifgalxkzv2\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_869\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_870\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-jz3cqifgalxkzv2","anchor":"5332792"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rqh3hjiav6yopjr\/","ref":"T215531#5332817"},[],{"anchor":"5332817"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_291\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_876\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_877\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_293\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rqh3hjiav6yopjr\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_878\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_879\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-rqh3hjiav6yopjr","anchor":"5332817"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-xopah5clof3pzhx\/","ref":"T215531#5333564"},[],{"anchor":"5333564"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_295\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_885\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_886\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_297\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-xopah5clof3pzhx\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_887\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_888\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-xopah5clof3pzhx","anchor":"5333564"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-4q4kk5an6gfu3ok\/","ref":"T215531#5333793"},[],{"anchor":"5333793"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_299\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_894\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_895\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_301\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-4q4kk5an6gfu3ok\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_896\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_897\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-4q4kk5an6gfu3ok","anchor":"5333793"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-7a2ol45yum73qqa\/","ref":"T215531#5333987"},[],{"anchor":"5333987"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_303\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_903\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_904\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_305\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-7a2ol45yum73qqa\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_905\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_906\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-7a2ol45yum73qqa","anchor":"5333987"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dqbozzyycbssimc\/","ref":"T215531#5333998"},[],{"anchor":"5333998"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_307\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_912\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_913\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_309\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dqbozzyycbssimc\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_914\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_915\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dqbozzyycbssimc","anchor":"5333998"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-uzfiwnjoiilx5qq\/","ref":"T215531#5334353"},[],{"anchor":"5334353"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_311\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_921\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_922\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_313\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-uzfiwnjoiilx5qq\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_923\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_924\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-uzfiwnjoiilx5qq","anchor":"5334353"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-5hkkyxbkb4sbsia\/","ref":"T215531#5334365"},[],{"anchor":"5334365"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_315\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_930\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_931\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_317\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-5hkkyxbkb4sbsia\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_932\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_933\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-5hkkyxbkb4sbsia","anchor":"5334365"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-syaopppyszetno6\/","ref":"T215531#5334479"},[],{"anchor":"5334479"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_319\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_939\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_940\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_321\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-syaopppyszetno6\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_941\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_942\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-syaopppyszetno6","anchor":"5334479"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-kj2qwslt4is5h6v\/","ref":"T215531#5334533"},[],{"anchor":"5334533"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_323\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_948\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_949\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_325\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-kj2qwslt4is5h6v\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_950\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_951\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-kj2qwslt4is5h6v","anchor":"5334533"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-glz4hmvqhzmtbai\/","ref":"T215531#5334856"},[],{"anchor":"5334856"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_327\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_957\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_958\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_329\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-glz4hmvqhzmtbai\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_959\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_960\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-glz4hmvqhzmtbai","anchor":"5334856"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-j527rjqddrpphql\/","ref":"T215531#5334868"},[],{"anchor":"5334868"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_331\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_966\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_967\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_333\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-j527rjqddrpphql\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_968\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_969\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-j527rjqddrpphql","anchor":"5334868"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-s2ls26vdzybja75\/","ref":"T215531#5335032"},[],{"anchor":"5335032"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_335\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_975\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_976\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_337\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-s2ls26vdzybja75\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_977\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_978\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-s2ls26vdzybja75","anchor":"5335032"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rxaliuwduwmnfr5\/","ref":"T215531#5335033"},[],{"anchor":"5335033"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_339\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_984\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_985\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_341\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rxaliuwduwmnfr5\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_986\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_987\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rxaliuwduwmnfr5","anchor":"5335033"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-u7a6nnnertcja3v\/","ref":"T215531#5335083"},[],{"anchor":"5335083"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_343\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_993\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_994\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_345\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-u7a6nnnertcja3v\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_995\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_996\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-u7a6nnnertcja3v","anchor":"5335083"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-rxpnnpmfdpzgqx4\/","ref":"T215531#5335085"},[],{"anchor":"5335085"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_347\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1002\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1003\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_349\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-rxpnnpmfdpzgqx4\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1004\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1005\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-rxpnnpmfdpzgqx4","anchor":"5335085"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-we7glrklo4z5gzv\/","ref":"T215531#5335106"},[],{"anchor":"5335106"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_351\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1011\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1012\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_353\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-we7glrklo4z5gzv\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1013\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1014\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-we7glrklo4z5gzv","anchor":"5335106"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-lmnpubqmq725i3m\/","ref":"T215531#5335114"},[],{"anchor":"5335114"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_355\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1020\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1021\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_357\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-lmnpubqmq725i3m\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1022\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1023\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-lmnpubqmq725i3m","anchor":"5335114"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-gu5iox6bfrl74t3\/","ref":"T215531#5335116"},[],{"anchor":"5335116"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_359\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1029\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1030\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_361\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-gu5iox6bfrl74t3\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1031\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1032\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Conduit"},[],{"phid":"PHID-XACT-TASK-gu5iox6bfrl74t3","anchor":"5335116"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-yosruzjhjjztszz\/","ref":"T215531#5335247"},[],{"anchor":"5335247"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_363\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1038\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1039\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_365\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-yosruzjhjjztszz\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1040\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1041\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-yosruzjhjjztszz","anchor":"5335247"},{"targetID":"UQ0_1","uri":"\/transactions\/quote\/PHID-XACT-TASK-dmx3a52gwdnvp5j\/","ref":"T215531#5335262"},[],{"anchor":"5335262"},[],{"items":"\u003cul class=\"phabricator-action-list-view \"\u003e\u003cli id=\"UQ0_367\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"#\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-quote\" data-meta=\"0_1047\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-quote-left phabricator-action-view-icon\" data-meta=\"0_1048\" aria-hidden=\"true\"\u003e\u003c\/span\u003eQuote Comment\u003c\/a\u003e\u003c\/li\u003e\u003cli id=\"UQ0_369\" class=\"phabricator-action-view phabricator-action-view-href action-has-icon\" style=\"\"\u003e\u003ca href=\"\/transactions\/raw\/PHID-XACT-TASK-dmx3a52gwdnvp5j\/\" class=\"phabricator-action-view-item\" data-sigil=\"transaction-raw\" data-meta=\"0_1049\"\u003e\u003cspan class=\"visual-only phui-icon-view phui-font-fa fa-code phabricator-action-view-icon\" data-meta=\"0_1050\" aria-hidden=\"true\"\u003e\u003c\/span\u003eView Raw Remarkup\u003c\/a\u003e\u003c\/li\u003e\u003c\/ul\u003e"},[],{"tip":"Via Web"},[],{"phid":"PHID-XACT-TASK-dmx3a52gwdnvp5j","anchor":"5335262"}],"javelin_behaviors":{"phui-hovercards":[],"phui-dropdown-menu":[],"phabricator-watch-anchor":[],"phabricator-tooltips":[]},"javelin_resources":["https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/2eeda9e0\/core.pkg.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/98e6504a\/rsrc\/externals\/javelin\/core\/init.js","https:\/\/phab.wmfusercontent.org\/res\/defaultX\/phabricator\/968d91ee\/core.pkg.css"]}