Page MenuHomePhabricator
Paste P8701

zuul-server kill -SIGUSR2 (threaddump)
ActivePublic

Authored by hashar on Jul 2 2019, 4:49 PM.
2019-07-02 16:48:26,748 DEBUG zuul.stack_dump: Thread: 139978193487616
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978185094912
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139977681794816
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 128, in run
self._handleEvent()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 116, in _handleEvent
event.change_number, event.patch_number, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 345, in _updateChange
dep = self._getChange(dep_num, dep_ps, refresh=True)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 305, in _updateChange
change):
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 210, in _getDependsOnFromCommit
records.extend(self.connection.simpleQuery(query))
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 348, in simpleQuery
chunk, more_changes = _query_chunk(query)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 320, in _query_chunk
out, err = self._ssh(cmd)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 388, in _ssh
out = stdout.read()
File "/usr/lib/python2.7/dist-packages/paramiko/file.py", line 150, in read
new_data = self._read(self._DEFAULT_BUFSIZE)
File "/usr/lib/python2.7/dist-packages/paramiko/channel.py", line 1217, in _read
return self.channel.recv(size)
File "/usr/lib/python2.7/dist-packages/paramiko/channel.py", line 596, in recv
out = self.in_buffer.read(nbytes, self.timeout)
File "/usr/lib/python2.7/dist-packages/paramiko/buffered_pipe.py", line 147, in read
self._cv.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978755536640
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop
ret = poll.poll()
Thread: 139979045463808
File "/usr/bin/zuul-server", line 10, in <module>
sys.exit(main())
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 239, in main
server.main()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 211, in main
signal.pause()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/__init__.py", line 40, in stack_dump_handler
log_str += "".join(traceback.format_stack(stack_frame))
Thread: 139978705180416
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978161956608
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 204, in run
self._run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 180, in _run
self._listen(stdout, stderr)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 160, in _listen
ret = poll.poll(self.poll_timeout)
Thread: 139978713573120
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978721965824
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978210273024
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978201880320
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139977631590144
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop
ret = poll.poll()
Thread: 139977639982848
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/webapp.py", line 61, in run
self.server.serve_forever()
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 1084, in serve_forever
self.handle_request()
File "/usr/lib/python2.7/SocketServer.py", line 276, in handle_request
fd_sets = _eintr_retry(select.select, [self], [], [], timeout)
File "/usr/lib/python2.7/SocketServer.py", line 155, in _eintr_retry
return func(*args)
Thread: 139977669089024
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1627, in run
ptype, m = self.packetizer.read_message()
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 341, in read_message
header = self.read_all(self.__block_size_in, check_rekey=True)
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 204, in read_all
x = self.__socket.recv(n)
Thread: 139978218665728
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978747143936
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139977228015360
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139977279141632
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/rpclistener.py", line 68, in run
job = self.worker.getJob()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 1898, in getJob
job = self.job_queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978738751232
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1627, in run
ptype, m = self.packetizer.read_message()
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 341, in read_message
header = self.read_all(self.__block_size_in, check_rekey=True)
File "/usr/lib/python2.7/dist-packages/paramiko/packet.py", line 204, in read_all
x = self.__socket.recv(n)
Thread: 139977623197440
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978979469056
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 744, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978170349312
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/scheduler.py", line 968, in run
self.wake_event.wait()
File "/usr/lib/python2.7/threading.py", line 621, in wait
self.__cond.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978971076352
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/launcher/gearman.py", line 47, in run
self.wake_event.wait(300)
File "/usr/lib/python2.7/threading.py", line 621, in wait
self.__cond.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 359, in wait
_sleep(delay)
Thread: 139977270748928
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 340, in wait
waiter.acquire()
Thread: 139978987861760
File "/usr/lib/python2.7/threading.py", line 783, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 830, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 856, in _pollLoop
ret = poll.poll()