Page MenuHomePhabricator
Paste P8357

zuul kill -SIGUSR2
ActivePublic

Authored by hashar on Apr 5 2019, 9:28 PM.
Tags
None
Referenced Files
F28585480: raw.txt
Apr 5 2019, 9:28 PM
Subscribers
None
2019-04-05 21:27:50,845 DEBUG zuul.stack_dump: Thread: 140422873605888
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422865213184
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422881998592
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140421840193280
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140421848585984
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop
ret = poll.poll()
Thread: 140421856978688
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/webapp.py", line 61, in run
self.server.serve_forever()
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 1084, in serve_forever
self.handle_request()
File "/usr/lib/python2.7/SocketServer.py", line 276, in handle_request
fd_sets = _eintr_retry(select.select, [self], [], [], timeout)
File "/usr/lib/python2.7/SocketServer.py", line 155, in _eintr_retry
return func(*args)
Thread: 140423330625280
File "/usr/bin/zuul-server", line 10, in <module>
sys.exit(main())
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 239, in main
server.main()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 211, in main
signal.pause()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/__init__.py", line 40, in stack_dump_handler
log_str += "".join(traceback.format_stack(stack_frame))
Thread: 140422831642368
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422840035072
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422848427776
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422269626112
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422278018816
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140421831800576
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/rpclistener.py", line 68, in run
job = self.worker.getJob()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 1898, in getJob
job = self.job_queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140423230793472
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140423239186176
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140423247578880
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop
ret = poll.poll()
Thread: 140423255971584
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/launcher/gearman.py", line 47, in run
self.wake_event.wait(300)
File "/usr/lib/python2.7/threading.py", line 621, in wait
self.__cond.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 359, in wait
_sleep(delay)
Thread: 140421823407872
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1627, in run
ptype, m = self.packetizer.read_message()
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 341, in read_message
header = self.read_all(self.__block_size_in, check_rekey=True)
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 204, in read_all
x = self.__socket.recv(n)
Thread: 140423264626432
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140423273019136
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop
ret = poll.poll()
Thread: 140421318571776
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422238095104
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 128, in run
self._handleEvent()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 116, in _handleEvent
event.change_number, event.patch_number, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 238, in _updateChange
data = self.connection.query(change.number)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 291, in query
out, err = self._ssh(cmd)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 378, in _ssh
out = stdout.read()
File "/usr/lib/python2.7/dist-packages/paramiko/file.py", line 150, in read
new_data = self._read(self._DEFAULT_BUFSIZE)
File "/usr/lib/python2.7/dist-packages/paramiko/channel.py", line 1217, in _read
return self.channel.recv(size)
File "/usr/lib/python2.7/dist-packages/paramiko/channel.py", line 596, in recv
out = self.in_buffer.read(nbytes, self.timeout)
File "/usr/lib/python2.7/dist-packages/paramiko/buffered_pipe.py", line 147, in read
self._cv.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422246487808
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 204, in run
self._run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 180, in _run
self._listen(stdout, stderr)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 160, in _listen
ret = poll.poll(self.poll_timeout)
Thread: 140422254880512
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/scheduler.py", line 968, in run
self.wake_event.wait()
File "/usr/lib/python2.7/threading.py", line 621, in wait
self.__cond.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 140422225389312
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1627, in run
ptype, m = self.packetizer.read_message()
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 341, in read_message
header = self.read_all(self.__block_size_in, check_rekey=True)
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 204, in read_all
x = self.__socket.recv(n)