All five of the cloudvirt nodes are currently failing to launch VMs. This seems to have regressed during the Rocky upgrade.
fault | {'code': 500, 'created': '2020-05-14T16:17:18Z', 'message': 'internal error: qemu unexpectedly closed the monitor: Failed to open module: /usr/lib/x86_64-linux-gnu/qemu/block-rbd.so: undefined symbol: qobject_input_visitor_new_keyval\n2020-05-14T16:17:16.851857Z qemu-system-x86_64: -drive file=rbd:compute/ba4ba70d-6', 'details': 'Traceback (most recent call last):\n File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2132, in _build_and_run_instance\n block_device_info=block_device_info)\n File "/usr/lib/python3/dist-packages/nova/virt/libvirt/driver.py", line 3080, in spawn\n destroy_disks_on_failure=True)\n File "/usr/lib/python3/dist-packages/nova/virt/libvirt/driver.py", line 5646, in _create_domain_and_network\n destroy_disks_on_failure)\n File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__\n self.force_reraise()\n File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type_, self.value, self.tb)\n File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise\n raise value\n File "/usr/lib/python3/dist-packages/nova/virt/libvirt/driver.py", line 5615, in _create_domain_and_network\n post_xml_callback=post_xml_callback)\n File "/usr/lib/python3/dist-packages/nova/virt/libvirt/driver.py", line 5550, in _create_domain\n guest.launch(pause=pause)\n File "/usr/lib/python3/dist-packages/nova/virt/libvirt/guest.py", line 144, in launch\n self._encoded_xml, errors=\'ignore\')\n File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__\n self.force_reraise()\n File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type_, self.value, self.tb)\n File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise\n raise value\n File "/usr/lib/python3/dist-packages/nova/virt/libvirt/guest.py", line 139, in launch\n return self._domain.createWithFlags(flags)\n File "/usr/lib/python3/dist-packages/eventlet/tpool.py", line 186, in doit\n result = proxy_call(self._autowrap, f, *args, **kwargs)\n File "/usr/lib/python3/dist-packages/eventlet/tpool.py", line 144, in proxy_call\n rv = execute(f, *args, **kwargs)\n File "/usr/lib/python3/dist-packages/eventlet/tpool.py", line 125, in execute\n six.reraise(c, e, tb)\n File "/usr/lib/python3/dist-packages/eventlet/support/six.py", line 625, in reraise\n raise value\n File "/usr/lib/python3/dist-packages/eventlet/tpool.py", line 83, in tworker\n rv = meth(*args, **kwargs)\n File "/usr/lib/python3/dist-packages/libvirt.py", line 1090, in createWithFlags\n if ret == -1: raise libvirtError (\'virDomainCreateWithFlags() failed\', dom=self)\nlibvirt.libvirtError: internal error: qemu unexpectedly closed the monitor: Failed to open module: /usr/lib/x86_64-linux-gnu/qemu/block-rbd.so: undefined symbol: qobject_input_visitor_new_keyval\n2020-05-14T16:17:16.851857Z qemu-system-x86_64: -drive file=rbd:compute/ba4ba70d-6773-49e8-bf26-e3b6f8a9fb9d_disk:id=eqiad1-compute:auth_supported=cephx\\;none:mon_host=208.80.154.148\\:6789\\;208.80.154.149\\:6789\\;208.80.154.150\\:6789,file.password-secret=virtio-disk0-secret0,format=raw,if=none,id=drive-virtio-disk0,cache=writeback,throttling.bps-total=250000000,throttling.iops-total=250: Unknown protocol \'rbd\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 1940, in _do_build_and_run_instance\n filter_properties, request_spec)\n File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2229, in _build_and_run_instance\n instance_uuid=instance.uuid, reason=six.text_type(e))\nnova.exception.RescheduledException: Build of instance ba4ba70d-6773-49e8-bf26-e3b6f8a9fb9d was re-scheduled: internal error: qemu unexpectedly closed the monitor: Failed to open module: /usr/lib/x86_64-linux-gnu/qemu/block-rbd.so: undefined symbol: qobject_input_visitor_new_keyval\n2020-05-14T16:17:16.851857Z qemu-system-x86_64: -drive file=rbd:compute/ba4ba70d-6773-49e8-bf26-e3b6f8a9fb9d_disk:id=eqiad1-compute:auth_supported=cephx\\;none:mon_host=208.80.154.148\\:6789\\;208.80.154.149\\:6789\\;208.80.154.150\\:6789,file.password-secret=virtio-disk0-secret0,format=raw,if=none,id=drive-virtio-disk0,cache=writeback,throttling.bps-total=250000000,throttling.iops-total=250: Unknown protocol \'rbd\'\n'}