Event Timeline
import collections from keystoneclient.session import Session as KeystoneSession from keystoneclient.auth.identity.v3 import Password as KeystonePassword from keystoneclient.v3 import Client as KeystoneClient from novaclient import client as novaclient from glanceclient import client as glanceclient def get_keystone_session(project): return KeystoneSession(auth=KeystonePassword( auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3", username="novaobserver", password='Fs6Dq2RtG8KwmM2Z', # public guest password project_name=project, user_domain_name='default', project_domain_name='default' )) keystone_client = KeystoneClient( session=get_keystone_session('bastion'), endpoint="http://cloudcontrol1003.wikimedia.org:5000/v3", interface='public' ) for project in keystone_client.projects.list(): if project.name != 'admin': session = get_keystone_session(project.name) for region in ['eqiad', 'eqiad1-r']: nclient = novaclient.Client("2.0", session=session, region_name=region) image_instances = collections.defaultdict(list) for s in nclient.servers.list(): image_instances[s.image['id']].append(s.name) gclient = glanceclient.Client(version=2, session=session, region_name=region) existing_images = {} for image in gclient.images.list(): existing_images[image.id] = image.name for missing_image in set(image_instances.keys()) - set(existing_images.keys()): print(project.name, region, missing_image, image_instances[missing_image])
A couple of different things are happening here. One is that in many cases the image exists but is inaccessible to the project using it. I'm fixing those one by one.
The other issue is that gclient.images.list() returns owned images but doesn't return private images that are visible to the current project but not public or owned by that project. I don't know why, but the upcoming code addresses this, badly.
import collections from keystoneclient.session import Session as KeystoneSession from keystoneclient.auth.identity.v3 import Password as KeystonePassword from keystoneclient.v3 import Client as KeystoneClient from novaclient import client as novaclient from glanceclient import client as glanceclient def get_keystone_session(project): return KeystoneSession(auth=KeystonePassword( auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3", username="novaobserver", password='Fs6Dq2RtG8KwmM2Z', # public guest password project_name=project, user_domain_name='default', project_domain_name='default' )) keystone_client = KeystoneClient( session=get_keystone_session('bastion'), endpoint="http://cloudcontrol1003.wikimedia.org:5000/v3", interface='public' ) for project in keystone_client.projects.list(): if project.name != 'admin': session = get_keystone_session(project.name) for region in ['eqiad', 'eqiad1-r']: missing_images=[] nclient = novaclient.Client("2.0", session=session, region_name=region) image_instances = collections.defaultdict(list) for s in nclient.servers.list(): image_instances[s.image['id']].append(s.name) gclient = glanceclient.Client(version=2, session=session, region_name=region) existing_images = {} for image in image_instances: try: image_obj = gclient.images.get(image) except: missing_images.append(image) for missing_image in missing_images: print(project.name, region, missing_image, image_instances[missing_image])
$ python ./findDodgyInstances.py
(u'maps', 'eqiad', u'45278e02-7898-4aa6-a8eb-98c677c6b60c', [u'overpass-wiki'])
(u'openstack', 'eqiad', u'f2ef20b4-0a51-4b0b-a4ea-2c7186fcb91c', [u'labs-bootstrapvz-stretch'])
(u'swift', 'eqiad', u'34964ef5-ac95-4e21-8f7d-ecf8e58978bb', [u'swift-stretch-ms-be02', u'swift-stretch-ms-be01', u'swift-stretch-ms-fe01'])
(u'wikistats', 'eqiad', u'34964ef5-ac95-4e21-8f7d-ecf8e58978bb', [u'wikistats-kraken'])