Apparent root cause of T308943: CI fails with 'This change or one of its cross-repo dependencies was unable to be automatically merged' for a lot of repos and T309371: Gerrit: all patches are being reported as merge conflicts
Max connection count for user jenkins-bot exceeded, rejecting new connection. currentSessionCount = 4, maxSessionCount = 4
Gerrit itself itself shows 3 open connections:
$ ssh -p 29418 gerrit.wikimedia.org gerrit show-connections | grep jenkins 716fc32f jenkins-bot contint2001.wikimedi 944d965d jenkins-bot contint2001.wikimedi f45092f9 jenkins-bot contint2001.wikimedi
@hashar took a stack dump (kill -SIGUSR2 <pidof zuul>)
1 | 2022-07-12 08:59:01,193 DEBUG zuul.stack_dump: Thread: 140118451013376 |
---|---|
2 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
3 | self.__bootstrap_inner() |
4 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
5 | self.run() |
6 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/transport.py", line 1780, in run |
7 | ptype, m = self.packetizer.read_message() |
8 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/packet.py", line 391, in read_message |
9 | header = self.read_all(self.__block_size_in, check_rekey=True) |
10 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/packet.py", line 254, in read_all |
11 | x = self.__socket.recv(n) |
12 | Thread: 140117645707008 |
13 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
14 | self.__bootstrap_inner() |
15 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
16 | self.run() |
17 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/transport.py", line 1780, in run |
18 | ptype, m = self.packetizer.read_message() |
19 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/packet.py", line 391, in read_message |
20 | header = self.read_all(self.__block_size_in, check_rekey=True) |
21 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/packet.py", line 254, in read_all |
22 | x = self.__socket.recv(n) |
23 | Thread: 140117939320576 |
24 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
25 | self.__bootstrap_inner() |
26 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
27 | self.run() |
28 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 210, in run |
29 | self._run() |
30 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 186, in _run |
31 | self._listen(stdout, stderr) |
32 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 166, in _listen |
33 | ret = poll.poll(self.poll_timeout) |
34 | Thread: 140117930927872 |
35 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
36 | self.__bootstrap_inner() |
37 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
38 | self.run() |
39 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 134, in run |
40 | self._handleEvent() |
41 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 60, in _handleEvent |
42 | time.sleep(max((ts + self.delay) - now, 0.0)) |
43 | Thread: 140117922535168 |
44 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
45 | self.__bootstrap_inner() |
46 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
47 | self.run() |
48 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/transport.py", line 1780, in run |
49 | ptype, m = self.packetizer.read_message() |
50 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/packet.py", line 391, in read_message |
51 | header = self.read_all(self.__block_size_in, check_rekey=True) |
52 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paramiko/packet.py", line 254, in read_all |
53 | x = self.__socket.recv(n) |
54 | Thread: 140118476191488 |
55 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
56 | self.__bootstrap_inner() |
57 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
58 | self.run() |
59 | File "/usr/lib/python2.7/threading.py", line 754, in run |
60 | self.__target(*self.__args, **self.__kwargs) |
61 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
62 | runnable = self.queue.get() |
63 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
64 | self.not_empty.wait() |
65 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
66 | waiter.acquire() |
67 | Thread: 140119038207744 |
68 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
69 | self.__bootstrap_inner() |
70 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
71 | self.run() |
72 | File "/usr/lib/python2.7/threading.py", line 754, in run |
73 | self.__target(*self.__args, **self.__kwargs) |
74 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
75 | runnable = self.queue.get() |
76 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
77 | self.not_empty.wait() |
78 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
79 | waiter.acquire() |
80 | Thread: 140118425835264 |
81 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
82 | self.__bootstrap_inner() |
83 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
84 | self.run() |
85 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/scheduler.py", line 968, in run |
86 | self.wake_event.wait() |
87 | File "/usr/lib/python2.7/threading.py", line 614, in wait |
88 | self.__cond.wait(timeout) |
89 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
90 | waiter.acquire() |
91 | Thread: 140118434227968 |
92 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
93 | self.__bootstrap_inner() |
94 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
95 | self.run() |
96 | File "/usr/lib/python2.7/threading.py", line 754, in run |
97 | self.__target(*self.__args, **self.__kwargs) |
98 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
99 | runnable = self.queue.get() |
100 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
101 | self.not_empty.wait() |
102 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
103 | waiter.acquire() |
104 | Thread: 140117637314304 |
105 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
106 | self.__bootstrap_inner() |
107 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
108 | self.run() |
109 | File "/usr/lib/python2.7/threading.py", line 754, in run |
110 | self.__target(*self.__args, **self.__kwargs) |
111 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
112 | runnable = self.queue.get() |
113 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
114 | self.not_empty.wait() |
115 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
116 | waiter.acquire() |
117 | Thread: 140117914142464 |
118 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
119 | self.__bootstrap_inner() |
120 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
121 | self.run() |
122 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/webapp.py", line 61, in run |
123 | self.server.serve_forever() |
124 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 1084, in serve_forever |
125 | self.handle_request() |
126 | File "/usr/lib/python2.7/SocketServer.py", line 274, in handle_request |
127 | fd_sets = _eintr_retry(select.select, [self], [], [], timeout) |
128 | File "/usr/lib/python2.7/SocketServer.py", line 150, in _eintr_retry |
129 | return func(*args) |
130 | Thread: 140118459406080 |
131 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
132 | self.__bootstrap_inner() |
133 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
134 | self.run() |
135 | File "/usr/lib/python2.7/threading.py", line 754, in run |
136 | self.__target(*self.__args, **self.__kwargs) |
137 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
138 | runnable = self.queue.get() |
139 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
140 | self.not_empty.wait() |
141 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
142 | waiter.acquire() |
143 | Thread: 140117662492416 |
144 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
145 | self.__bootstrap_inner() |
146 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
147 | self.run() |
148 | File "/usr/lib/python2.7/threading.py", line 754, in run |
149 | self.__target(*self.__args, **self.__kwargs) |
150 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 769, in _doConnectLoop |
151 | self.connections_condition.wait() |
152 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
153 | waiter.acquire() |
154 | Thread: 140117670885120 |
155 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
156 | self.__bootstrap_inner() |
157 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
158 | self.run() |
159 | File "/usr/lib/python2.7/threading.py", line 754, in run |
160 | self.__target(*self.__args, **self.__kwargs) |
161 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 855, in _doPollLoop |
162 | self._pollLoop() |
163 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 881, in _pollLoop |
164 | ret = poll.poll() |
165 | Thread: 140119046600448 |
166 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
167 | self.__bootstrap_inner() |
168 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
169 | self.run() |
170 | File "/usr/lib/python2.7/threading.py", line 754, in run |
171 | self.__target(*self.__args, **self.__kwargs) |
172 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
173 | runnable = self.queue.get() |
174 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
175 | self.not_empty.wait() |
176 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
177 | waiter.acquire() |
178 | Thread: 140118467798784 |
179 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
180 | self.__bootstrap_inner() |
181 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
182 | self.run() |
183 | File "/usr/lib/python2.7/threading.py", line 754, in run |
184 | self.__target(*self.__args, **self.__kwargs) |
185 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
186 | runnable = self.queue.get() |
187 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
188 | self.not_empty.wait() |
189 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
190 | waiter.acquire() |
191 | Thread: 140117654099712 |
192 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
193 | self.__bootstrap_inner() |
194 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
195 | self.run() |
196 | File "/usr/lib/python2.7/threading.py", line 754, in run |
197 | self.__target(*self.__args, **self.__kwargs) |
198 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/rpclistener.py", line 68, in run |
199 | job = self.worker.getJob() |
200 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 2092, in getJob |
201 | job = self.job_queue.get() |
202 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
203 | self.not_empty.wait() |
204 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
205 | waiter.acquire() |
206 | Thread: 140118442620672 |
207 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
208 | self.__bootstrap_inner() |
209 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
210 | self.run() |
211 | File "/usr/lib/python2.7/threading.py", line 754, in run |
212 | self.__target(*self.__args, **self.__kwargs) |
213 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
214 | runnable = self.queue.get() |
215 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
216 | self.not_empty.wait() |
217 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
218 | waiter.acquire() |
219 | Thread: 140119187257152 |
220 | File "/srv/deployment/zuul/venv/bin/zuul-server", line 10, in <module> |
221 | sys.exit(main()) |
222 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 239, in main |
223 | server.main() |
224 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 211, in main |
225 | signal.pause() |
226 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/cmd/__init__.py", line 42, in stack_dump_handler |
227 | log_str += "".join(traceback.format_stack(stack_frame)) |
228 | Thread: 140119054993152 |
229 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
230 | self.__bootstrap_inner() |
231 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
232 | self.run() |
233 | File "/usr/lib/python2.7/threading.py", line 754, in run |
234 | self.__target(*self.__args, **self.__kwargs) |
235 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
236 | runnable = self.queue.get() |
237 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
238 | self.not_empty.wait() |
239 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
240 | waiter.acquire() |
241 | Thread: 140119029815040 |
242 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
243 | self.__bootstrap_inner() |
244 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
245 | self.run() |
246 | File "/usr/lib/python2.7/threading.py", line 754, in run |
247 | self.__target(*self.__args, **self.__kwargs) |
248 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/paste/httpserver.py", line 866, in worker_thread_callback |
249 | runnable = self.queue.get() |
250 | File "/usr/lib/python2.7/Queue.py", line 168, in get |
251 | self.not_empty.wait() |
252 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
253 | waiter.acquire() |
254 | Thread: 140119080171264 |
255 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
256 | self.__bootstrap_inner() |
257 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
258 | self.run() |
259 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/zuul/launcher/gearman.py", line 47, in run |
260 | self.wake_event.wait(300) |
261 | File "/usr/lib/python2.7/threading.py", line 614, in wait |
262 | self.__cond.wait(timeout) |
263 | File "/usr/lib/python2.7/threading.py", line 359, in wait |
264 | _sleep(delay) |
265 | Thread: 140119167018752 |
266 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
267 | self.__bootstrap_inner() |
268 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
269 | self.run() |
270 | File "/usr/lib/python2.7/threading.py", line 754, in run |
271 | self.__target(*self.__args, **self.__kwargs) |
272 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 855, in _doPollLoop |
273 | self._pollLoop() |
274 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 881, in _pollLoop |
275 | ret = poll.poll() |
276 | Thread: 140119063385856 |
277 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
278 | self.__bootstrap_inner() |
279 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
280 | self.run() |
281 | File "/usr/lib/python2.7/threading.py", line 754, in run |
282 | self.__target(*self.__args, **self.__kwargs) |
283 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 769, in _doConnectLoop |
284 | self.connections_condition.wait() |
285 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
286 | waiter.acquire() |
287 | Thread: 140119071778560 |
288 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
289 | self.__bootstrap_inner() |
290 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
291 | self.run() |
292 | File "/usr/lib/python2.7/threading.py", line 754, in run |
293 | self.__target(*self.__args, **self.__kwargs) |
294 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 855, in _doPollLoop |
295 | self._pollLoop() |
296 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 881, in _pollLoop |
297 | ret = poll.poll() |
298 | Thread: 140119158626048 |
299 | File "/usr/lib/python2.7/threading.py", line 774, in __bootstrap |
300 | self.__bootstrap_inner() |
301 | File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner |
302 | self.run() |
303 | File "/usr/lib/python2.7/threading.py", line 754, in run |
304 | self.__target(*self.__args, **self.__kwargs) |
305 | File "/srv/deployment/zuul/venv/local/lib/python2.7/site-packages/gear/__init__.py", line 769, in _doConnectLoop |
306 | self.connections_condition.wait() |
307 | File "/usr/lib/python2.7/threading.py", line 340, in wait |
308 | waiter.acquire() |
309 |
The three connections:
lsof -p 15751|grep gerrit zuul-serv 15751 zuul 8u IPv6 295859537 0t0 TCP contint2001.wikimedia.org:57134->gerrit.wikimedia.org:29418 (ESTABLISHED) zuul-serv 15751 zuul 17u IPv6 173274822 0t0 TCP contint2001.wikimedia.org:39456->gerrit.wikimedia.org:29418 (ESTABLISHED) zuul-serv 15751 zuul 24u IPv6 295859542 0t0 TCP contint2001.wikimedia.org:57136->gerrit.wikimedia.org:29418 (ESTABLISHED)
We should probably figure out why it's happening, and maybe see if setting up an icinga alert is possible?