Page MenuHomePhabricator
Paste P3204

Zuul stall establishing ssh connection to Gerrit
ActivePublic
Actions

Authored by hashar on Jun 2 2016, 1:33 PM.
Tags
None
Referenced Files
F4103652: Zuul stall establishing ssh connection to Gerrit
Jun 2 2016, 2:09 PM
F4103496:
Jun 2 2016, 1:33 PM
Subscribers
None
Thread: 140230061446912
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1510, in run
Random.atfork()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/__init__.py", line 37, in atfork
_UserFriendlyRNG.reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 209, in reinit
_get_singleton().reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 154, in reinit
self._lock.acquire()
Thread: 140230069839616
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/rpclistener.py", line 76, in run
f(job)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/rpclistener.py", line 118, in handle_enqueue
pipeline.source.getChange(event, project)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 150, in getChange
refresh=refresh)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 173, in _getChange
self._updateChange(change, history)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/source/gerrit.py", line 238, in _updateChange
data = self.connection.query(change.number)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 287, in query
out, err = self._ssh(cmd)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 362, in _ssh
self._open()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 357, in _open
key_filename=self.keyfile)
File "/usr/lib/python2.7/dist-packages/paramiko/client.py", line 301, in connect
t.start_client()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 455, in start_client
Random.atfork()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/__init__.py", line 37, in atfork
_UserFriendlyRNG.reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 209, in reinit
_get_singleton().reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 154, in reinit
self._lock.acquire()
Thread: 140231151974144
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140230564747008
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 821, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 847, in _pollLoop
ret = poll.poll()
Thread: 140231772464896
File "/usr/bin/zuul-server", line 10, in <module>
sys.exit(main())
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 227, in main
server.main()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/server.py", line 196, in main
signal.pause()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/cmd/__init__.py", line 42, in stack_dump_handler
log_str += "".join(traceback.format_stack(stack_frame))
Thread: 140231101617920
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231110010624
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231118403328
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231675930368
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 739, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140230589925120
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 126, in run
self._handleEvent()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 48, in _handleEvent
ts, data = self.connection.getEvent()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 263, in getEvent
return self.event_queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140230598317824
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 201, in run
self._run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/connection/gerrit.py", line 173, in _run
key_filename=self.keyfile)
File "/usr/lib/python2.7/dist-packages/paramiko/client.py", line 301, in connect
t.start_client()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 455, in start_client
Random.atfork()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/__init__.py", line 37, in atfork
_UserFriendlyRNG.reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 209, in reinit
_get_singleton().reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 156, in reinit
return _UserFriendlyRNG.reinit(self)
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 94, in reinit
self._ec.reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 62, in reinit
block = self._osrng.read(32*32)
File "/usr/lib/python2.7/dist-packages/Crypto/Random/OSRNG/rng_base.py", line 76, in read
data = self._read(N)
Thread: 140230606710528
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/scheduler.py", line 947, in run
self.wake_event.wait()
File "/usr/lib/python2.7/threading.py", line 403, in wait
self.__cond.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140230581532416
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/dist-packages/paramiko/transport.py", line 1510, in run
Random.atfork()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/__init__.py", line 37, in atfork
_UserFriendlyRNG.reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 209, in reinit
_get_singleton().reinit()
File "/usr/lib/python2.7/dist-packages/Crypto/Random/_UserFriendlyRNG.py", line 154, in reinit
self._lock.acquire()
Thread: 140230615103232
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140230573139712
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/webapp.py", line 58, in run
self.server.serve_forever()
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 1084, in serve_forever
self.handle_request()
File "/usr/lib/python2.7/SocketServer.py", line 265, in handle_request
fd_sets = select.select([self], [], [], timeout)
Thread: 140231633966848
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231126796032
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140230078232320
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 739, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231135188736
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231642359552
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231650752256
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 739, in _doConnectLoop
self.connections_condition.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231659144960
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 821, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 847, in _pollLoop
ret = poll.poll()
Thread: 140231667537664
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/zuul/launcher/gearman.py", line 46, in run
self.wake_event.wait(300)
File "/usr/lib/python2.7/threading.py", line 403, in wait
self.__cond.wait(timeout)
File "/usr/lib/python2.7/threading.py", line 262, in wait
_sleep(delay)
Thread: 140231143581440
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/lib/python2.7/dist-packages/paste/httpserver.py", line 866, in worker_thread_callback
runnable = self.queue.get()
File "/usr/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File "/usr/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140231684323072
File "/usr/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 821, in _doPollLoop
self._pollLoop()
File "/usr/share/python/zuul/local/lib/python2.7/site-packages/gear/__init__.py", line 847, in _pollLoop
ret = poll.poll()
Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits