Page MenuHomePhabricator
Paste P3886

Stuck KVM instance
ActivePublic

Authored by chasemp on Aug 24 2016, 8:22 PM.
Tags
None
Referenced Files
F4397809: Stuck KVM instance
Aug 24 2016, 8:37 PM
F4397778: Stuck KVM instance
Aug 24 2016, 8:22 PM
F4397777: Stuck KVM instance
Aug 24 2016, 8:22 PM
Subscribers
None
We have instances that get stuck / freeze and can do no IO
Candidate: tools-exec-1217
Details:
44efcd75-808a-4af2-8c7a-8d70b703d7f2
labvirt1001.eqiad.wmnet
i-00000bde
Process info:
https://phabricator.wikimedia.org/P3886#18575
strace of (only) running subprocess:
https://phabricator.wikimedia.org/P3886#18576
gdb attach of running subprocess:
https://phabricator.wikimedia.org/P3886#18577
Active open files:
https://phabricator.wikimedia.org/P3886#18578
proc io information:
https://phabricator.wikimedia.org/P3886#18579

Event Timeline

libvirt+ 62648 1 99 Jun08 ? 102-15:40:33 /usr/bin/kvm -name i-00000bde -S -machine pc-i440fx-trusty,accel=kvm,usb=off -cpu SandyBridge,+erms,+smep,+fsgsbase,+pdpe1gb,+rdrand,+f16c,+osxsave,+dca,+pcid,+pdcm,+xtpr,+tm2,+est,+smx,+vmx,+ds_cpl,+monitor,+dtes64,+pbe,+tm,+ht,+ss,+acpi,+ds,+vme -m 8192 -realtime mlock=off -smp 4,sockets=4,cores=1,threads=1 -uuid 44efcd75-808a-4af2-8c7a-8d70b703d7f2 -smbios type=1,manufacturer=OpenStack Foundation,product=OpenStack Nova,version=2015.1.2,serial=35353636-3435-4d32-3235-313330334250,uuid=44efcd75-808a-4af2-8c7a-8d70b703d7f2 -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/i-00000bde.monitor,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=discard -no-hpet -no-shutdown -boot strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive file=/var/lib/nova/instances/44efcd75-808a-4af2-8c7a-8d70b703d7f2/disk,if=none,id=drive-virtio-disk0,format=qcow2,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=31,id=hostnet0,vhost=on,vhostfd=81 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=fa:16:3e:8d:df:2e,bus=pci.0,addr=0x3 -chardev file,id=charserial0,path=/var/lib/nova/instances/44efcd75-808a-4af2-8c7a-8d70b703d7f2/console.log -device isa-serial,chardev=charserial0,id=serial0 -chardev pty,id=charserial1 -device isa-serial,chardev=charserial1,id=serial1 -device usb-tablet,id=input0 -vnc 0.0.0.0:45 -k en-us -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -incoming fd:25 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5

top
62648 libvirt+ 20 0 12.743g 8.075g 16700 S 0.0 2.1 147826:16 kvm

root@labvirt1001:~# virsh domblkstat i-00000bde
rd_req 374577
rd_bytes 4057767936
wr_req 10191979
wr_bytes 364797059072
flush_operations 3039503
rd_total_times 2454549787955
wr_total_times 56178691942938
flush_total_times 1116955624610

This never changes. The instance can do no io

Status of child procs

root@labvirt1001:~# cat /proc/62648/task/**/status | grep -i state
State: S (sleeping)
State: S (sleeping)
State: S (sleeping)
State: S (sleeping)
State: R (running)
State: S (sleeping)

strace -f -p 62661 (running proc under stuck instance)

[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=5, revents=POLLIN}], left {0, 658997307})
[pid 62648] read(5, "\2\0\0\0\0\0\0\0", 16) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 0 (Timeout)
[pid 62648] read(5, 0x7ffc4c982a60, 16) = -1 EAGAIN (Resource temporarily unavailable)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 999000000}, NULL, 8) = 0 (Timeout)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 999000000}, NULL, 8) = 0 (Timeout)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 338200238}, NULL, 8) = 0 (Timeout)
[pid 62648] write(4, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=4, revents=POLLIN}], left {0, 658997590})
[pid 62648] write(5, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] read(4, "\1\0\0\0\0\0\0\0", 512) = 8
[pid 62648] write(5, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=5, revents=POLLIN}], left {0, 658998178})
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=5, revents=POLLIN}], left {0, 658998337})
[pid 62648] read(5, "\2\0\0\0\0\0\0\0", 16) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 0 (Timeout)
[pid 62648] read(5, 0x7ffc4c982a60, 16) = -1 EAGAIN (Resource temporarily unavailable)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 999000000}, NULL, 8) = 0 (Timeout)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 999000000}, NULL, 8) = 0 (Timeout)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 339097224}, NULL, 8) = 0 (Timeout)
[pid 62648] write(4, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=4, revents=POLLIN}], left {0, 658997168})
[pid 62648] write(5, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] read(4, "\1\0\0\0\0\0\0\0", 512) = 8
[pid 62648] write(5, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=5, revents=POLLIN}], left {0, 658997865})
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 1 ([{fd=5, revents=POLLIN}], left {0, 658998200})
[pid 62648] read(5, "\2\0\0\0\0\0\0\0", 16) = 8
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 659000000}, NULL, 8) = 0 (Timeout)
[pid 62648] read(5, 0x7ffc4c982a60, 16) = -1 EAGAIN (Resource temporarily unavailable)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 999000000}, NULL, 8) = 0 (Timeout)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 328678930}, NULL, 8) = 0 (Timeout)
[pid 62648] ioctl(10, KVM_IRQ_LINE_STATUS, 0x7ffc4c982a50) = 0
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 669000000}, NULL, 8 <unfinished ...>
[pid 62658] <... ioctl resumed> , 0)    = 0
[pid 62658] ioctl(10, KVM_IRQ_LINE_STATUS, 0x7f44e979fa30) = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] write(4, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] <... ppoll resumed> )       = 1 ([{fd=4, revents=POLLIN}], left {0, 662958094})
[pid 62658] ioctl(15, KVM_RUN <unfinished ...>
[pid 62648] write(5, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
[pid 62658] <... ioctl resumed> , 0)    = 0
[pid 62648] <... write resumed> )       = 8
[pid 62658] futex(0x7f44fa5e89a0, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
[pid 62648] read(4, "\1\0\0\0\0\0\0\0", 512) = 8
[pid 62648] write(5, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 62648] futex(0x7f44fa5e89a0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
[pid 62658] <... futex resumed> )       = 0
[pid 62648] <... futex resumed> )       = 1
[pid 62658] futex(0x7f44fa5e89a0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 663000000}, NULL, 8 <unfinished ...>
[pid 62658] <... futex resumed> )       = 0
[pid 62648] <... ppoll resumed> )       = 1 ([{fd=5, revents=POLLIN}], left {0, 662997364})
[pid 62658] ioctl(15, KVM_RUN <unfinished ...>
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 663000000}, NULL, 8 <unfinished ...>
[pid 62658] <... ioctl resumed> , 0)    = 0
[pid 62648] <... ppoll resumed> )       = 1 ([{fd=5, revents=POLLIN}], left {0, 662997909})
[pid 62658] ioctl(15, KVM_RUN <unfinished ...>
[pid 62648] read(5,  <unfinished ...>
[pid 62658] <... ioctl resumed> , 0)    = 0
[pid 62648] <... read resumed> "\2\0\0\0\0\0\0\0", 16) = 8
[pid 62658] futex(0x7f44fa5e89a0, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
[pid 62648] futex(0x7f44fa5e89a0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
[pid 62658] <... futex resumed> )       = -1 EAGAIN (Resource temporarily unavailable)
[pid 62648] <... futex resumed> )       = 0
[pid 62658] futex(0x7f44fa5e89a0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 663000000}, NULL, 8 <unfinished ...>
[pid 62658] <... futex resumed> )       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN, 0)       = 0
[pid 62658] ioctl(15, KVM_RUN <unfinished ...>
[pid 62648] <... ppoll resumed> )       = 0 (Timeout)
[pid 62648] read(5, 0x7ffc4c982a60, 16) = -1 EAGAIN (Resource temporarily unavailable)
[pid 62648] poll([{fd=8, events=POLLOUT}], 1, 0) = 1 ([{fd=8, revents=POLLOUT|POLLHUP}])
[pid 62648] ppoll([{fd=25, events=POLLIN|POLLERR|POLLHUP}, {fd=19, events=POLLIN|POLLERR|POLLHUP}, {fd=3, events=POLLIN|POLLERR|POLLHUP}, {fd=5, events=POLLIN}, {fd=20, events=POLLIN}, {fd=4, events=POLLIN}], 6, {0, 338454379}, NULL, 8Process 62661 detached
Process 62648 detached
 <detached ...>
Process 62658 detached
Process 62659 detached
Process 62660 detached
Process 62663 detached

GDB

(gdb) attach 62648
Attaching to process 62648
/usr/bin/qemu-system-x86_64 (deleted): No such file or directory.

what?

root@labvirt1001:~# gdb /proc/62648/exe 62648

root@labvirt1001:~# gdb /proc/62648/exe 62648
GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /proc/62648/exe...(no debugging symbols found)...done.
Attaching to program: /proc/62648/exe, process 62648

warning: .dynamic section for "/usr/lib/x86_64-linux-gnu/librbd.so.1" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/lib/x86_64-linux-gnu/librados.so.2" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/lib/x86_64-linux-gnu/libgcc_s.so.1" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/lib/x86_64-linux-gnu/libstdc++.so.6" is not at the expected address (wrong library or version mismatch?)

warning: Could not load shared library symbols for /usr/lib/x86_64-linux-gnu/libseccomp.so.2.
Do you need "set solib-search-path" or "set sysroot"?
Reading symbols from /lib/x86_64-linux-gnu/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libz.so.1
Reading symbols from /lib/x86_64-linux-gnu/libaio.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libaio.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4
Reading symbols from /usr/lib/x86_64-linux-gnu/librbd.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/librbd.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/librados.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/librados.so.2
Reading symbols from /lib/x86_64-linux-gnu/libglib-2.0.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libglib-2.0.so.0
Reading symbols from /lib/x86_64-linux-gnu/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/librt.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libpixman-1.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libpixman-1.so.0
Reading symbols from /lib/x86_64-linux-gnu/libutil.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libutil.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libnuma.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libnuma.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libbluetooth.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libbluetooth.so.3
Reading symbols from /lib/x86_64-linux-gnu/libncurses.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libncurses.so.5
Reading symbols from /lib/x86_64-linux-gnu/libtinfo.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libtinfo.so.5
Reading symbols from /lib/x86_64-linux-gnu/libbrlapi.so.0.6...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libbrlapi.so.0.6
Reading symbols from /usr/lib/x86_64-linux-gnu/libasound.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libasound.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libpulse.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libpulse.so.0
Reading symbols from /lib/x86_64-linux-gnu/libuuid.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libuuid.so.1
Reading symbols from /lib/x86_64-linux-gnu/libpng12.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libpng12.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libjpeg.so.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libjpeg.so.8
Reading symbols from /usr/lib/x86_64-linux-gnu/libsasl2.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libsasl2.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libgnutls.so.26...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libgnutls.so.26
Reading symbols from /usr/lib/x86_64-linux-gnu/libSDL-1.2.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libSDL-1.2.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libX11.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libX11.so.6
Reading symbols from /usr/lib/libxenstore.so.3.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libxenstore.so.3.0
Reading symbols from /usr/lib/libxenctrl-4.4.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libxenctrl-4.4.so
Reading symbols from /usr/lib/libxenguest-4.4.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libxenguest-4.4.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libfdt.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libfdt.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libspice-server.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libspice-server.so.1
Reading symbols from /lib/x86_64-linux-gnu/libusb-1.0.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libusb-1.0.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libusbredirparser.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libusbredirparser.so.1
Reading symbols from /lib/x86_64-linux-gnu/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libm.so.6
Reading symbols from /lib/x86_64-linux-gnu/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgcc_s.so.1
Reading symbols from /lib/x86_64-linux-gnu/libpthread.so.0...(no debugging symbols found)...done.

warning: File "/lib/x86_64-linux-gnu/libthread_db-1.0.so" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load".
To enable execution of this file add
	add-auto-load-safe-path /lib/x86_64-linux-gnu/libthread_db-1.0.so
line to your configuration file "/root/.gdbinit".
To completely disable this security protection add
	set auto-load safe-path /
line to your configuration file "/root/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual.  E.g., run from the shell:
---Type <return> to continue, or q <return> to quit---
	info "(gdb)Auto-loading safe path"

warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
Loaded symbols for /lib/x86_64-linux-gnu/libpthread.so.0
Reading symbols from /lib/x86_64-linux-gnu/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libc.so.6
Reading symbols from /usr/lib/x86_64-linux-gnu/libidn.so.11...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libidn.so.11
Reading symbols from /usr/lib/x86_64-linux-gnu/librtmp.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/librtmp.so.0
Reading symbols from /lib/x86_64-linux-gnu/libgcrypt.so.11...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgcrypt.so.11
Reading symbols from /usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/liblber-2.4.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/liblber-2.4.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libstdc++.so.6
Reading symbols from /lib/x86_64-linux-gnu/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libdl.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libnss3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libnss3.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libsmime3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libsmime3.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libnspr4.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libnspr4.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libboost_thread.so.1.54.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libboost_thread.so.1.54.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libboost_system.so.1.54.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libboost_system.so.1.54.0
Reading symbols from /lib/x86_64-linux-gnu/libpcre.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libpcre.so.3
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib/x86_64-linux-gnu/libjson-c.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libjson-c.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/pulseaudio/libpulsecommon-4.0.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/pulseaudio/libpulsecommon-4.0.so
Reading symbols from /lib/x86_64-linux-gnu/libdbus-1.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libdbus-1.so.3
Reading symbols from /usr/lib/x86_64-linux-gnu/libtasn1.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libtasn1.so.6
Reading symbols from /usr/lib/x86_64-linux-gnu/libp11-kit.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libp11-kit.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libpulse-simple.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libpulse-simple.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libXext.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libXext.so.6
Reading symbols from /usr/lib/x86_64-linux-gnu/libcaca.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libcaca.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libxcb.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libxcb.so.1
Reading symbols from /lib/x86_64-linux-gnu/liblzma.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/liblzma.so.5
Reading symbols from /lib/x86_64-linux-gnu/libssl.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libssl.so.1.0.0
Reading symbols from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
Reading symbols from /lib/x86_64-linux-gnu/libudev.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libudev.so.1
Reading symbols from /lib/x86_64-linux-gnu/libgpg-error.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgpg-error.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libkrb5.so.3
Reading symbols from /usr/lib/x86_64-linux-gnu/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libk5crypto.so.3
Reading symbols from /lib/x86_64-linux-gnu/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libcom_err.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libkrb5support.so.0
Reading symbols from /lib/x86_64-linux-gnu/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libresolv.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libgssapi.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libgssapi.so.3
Reading symbols from /usr/lib/x86_64-linux-gnu/libnssutil3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libnssutil3.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libplc4.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libplc4.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libplds4.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libplds4.so
Reading symbols from /lib/x86_64-linux-gnu/libwrap.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libwrap.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libsndfile.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libsndfile.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libasyncns.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libasyncns.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libffi.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libffi.so.6
Reading symbols from /lib/x86_64-linux-gnu/libslang.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libslang.so.2
Reading symbols from /lib/x86_64-linux-gnu/libncursesw.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libncursesw.so.5
Reading symbols from /usr/lib/x86_64-linux-gnu/libXau.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libXau.so.6
Reading symbols from /usr/lib/x86_64-linux-gnu/libXdmcp.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libXdmcp.so.6
Reading symbols from /lib/x86_64-linux-gnu/libcgmanager.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libcgmanager.so.0
Reading symbols from /lib/x86_64-linux-gnu/libnih.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libnih.so.1
Reading symbols from /lib/x86_64-linux-gnu/libnih-dbus.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libnih-dbus.so.1
Reading symbols from /lib/x86_64-linux-gnu/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libkeyutils.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libheimntlm.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libheimntlm.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libkrb5.so.26...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libkrb5.so.26
Reading symbols from /usr/lib/x86_64-linux-gnu/libasn1.so.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libasn1.so.8
Reading symbols from /usr/lib/x86_64-linux-gnu/libhcrypto.so.4...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libhcrypto.so.4
Reading symbols from /usr/lib/x86_64-linux-gnu/libroken.so.18...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libroken.so.18
Reading symbols from /lib/x86_64-linux-gnu/libnsl.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libnsl.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libFLAC.so.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libFLAC.so.8
Reading symbols from /usr/lib/x86_64-linux-gnu/libvorbisenc.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libvorbisenc.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/libvorbis.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libvorbis.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libogg.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libogg.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libwind.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libwind.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libheimbase.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libheimbase.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libhx509.so.5...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libhx509.so.5
Reading symbols from /usr/lib/x86_64-linux-gnu/libsqlite3.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libsqlite3.so.0
Reading symbols from /lib/x86_64-linux-gnu/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libcrypt.so.1
Reading symbols from /lib/x86_64-linux-gnu/libnss_compat.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libnss_compat.so.2
Reading symbols from /lib/x86_64-linux-gnu/libnss_nis.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libnss_nis.so.2
Reading symbols from /lib/x86_64-linux-gnu/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libnss_files.so.2
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/libanonymous.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/libanonymous.so
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/liblogin.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/liblogin.so
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/libntlm.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/libntlm.so
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/libcrammd5.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/libcrammd5.so
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/libplain.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/libplain.so
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/libdigestmd5.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/libdigestmd5.so
Reading symbols from /usr/lib/x86_64-linux-gnu/sasl2/libsasldb.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/sasl2/libsasldb.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libdb-5.3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libdb-5.3.so

warning: File "/lib/x86_64-linux-gnu/libthread_db-1.0.so" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load".

warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
0x00007f44f3c2c1ef in ?? () from /lib/x86_64-linux-gnu/libc.so.6
(gdb)
(gdb) bt
#0  0x00007f44f3c2c1ef in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007ffc4c982c40 in ?? ()
#2  0x00007ffc4c984f4c in ?? ()
#3  0x0000000000000000 in ?? ()
(gdb)

Active subprocess open files

root@labvirt1001:~# ls -al /proc/62661/fd/*
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/0 -> /dev/null
l-wx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/1 -> /var/log/libvirt/qemu/i-00000bde.log
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/10 -> anon_inode:kvm-vm
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/11 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/12 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/13 -> /var/lib/nova/instances/44efcd75-808a-4af2-8c7a-8d70b703d7f2/disk
lr-x------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/14 -> /var/lib/nova/instances/_base/7ef7aafc3814e1bd31983232b7440cb591fdd5e4
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/15 -> anon_inode:kvm-vcpu
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/16 -> anon_inode:kvm-vcpu
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/17 -> anon_inode:kvm-vcpu
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/18 -> anon_inode:kvm-vcpu
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/19 -> socket:[228990234]
l-wx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/2 -> /var/log/libvirt/qemu/i-00000bde.log
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/20 -> socket:[362120864]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/21 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/22 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/23 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/24 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/25 -> anon_inode:[eventfd]
lr-x------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/26 -> /dev/null
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/3 -> anon_inode:[signalfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/31 -> /dev/net/tun
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/4 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/5 -> anon_inode:[eventfd]
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/6 -> socket:[228990228]
l-wx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/7 -> /var/lib/nova/instances/44efcd75-808a-4af2-8c7a-8d70b703d7f2/console.log
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/8 -> /dev/ptmx
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/81 -> /dev/vhost-net
lrwx------ 1 libvirt-qemu kvm 64 Aug 24 20:16 /proc/62661/fd/9 -> /dev/kvm
root@labvirt1001:~#

root@labvirt1001:~# cat /proc/62661/io
rchar: 24352355657
wchar: 365885530056
syscr: 72172656
syscw: 138960102
read_bytes: 15484068352
write_bytes: 364834289664
cancelled_write_bytes: 0

All increment:

rchar: 24352355657
wchar: 365885530056
syscr: 72172656
syscw: 138960102

Does not:

read_bytes: 15484068352
write_bytes: 364834289664
cancelled_write_bytes: 0


Example:

root@labvirt1001:~# date && cat /proc/62661/io
Wed Aug 24 20:34:48 UTC 2016
rchar: 24352357881
wchar: 365885535661
syscr: 72172964
syscw: 138960640
read_bytes: 15484068352
write_bytes: 364834289664
cancelled_write_bytes: 0
root@labvirt1001:~# date && cat /proc/62661/io
Wed Aug 24 20:34:55 UTC 2016
rchar: 24352357929
wchar: 365885535733
syscr: 72172973
syscw: 138960649
read_bytes: 15484068352
write_bytes: 364834289664
cancelled_write_bytes: 0
root@labvirt1001:~# date && cat /proc/62661/io
Wed Aug 24 20:35:01 UTC 2016
rchar: 24352357961
wchar: 365885535781
syscr: 72172979
syscw: 138960655
read_bytes: 15484068352
write_bytes: 364834289664
cancelled_write_bytes: 0
root@labvirt1001:~#