Page MenuHomePhabricator
Paste P7843

instances with missing images
ActivePublic

Authored by Krenair on Nov 26 2018, 4:47 PM.
krenair@shinken-02:~/shinken-01-home$ python findDodgyInstances.py
(u'deployment-prep', 'eqiad', u'e9587207-1fa2-4faf-926b-d9efc41b176c', [u'deployment-zotero01'])
(u'discovery-stats', 'eqiad1-r', u'b3cfb043-ccdb-4f0e-961b-02149c9abb9d', [u'shinyserv-lb', u'shinyserv-01'])
(u'huggle', 'eqiad', u'e9587207-1fa2-4faf-926b-d9efc41b176c', [u'huggle-pg'])
(u'maps', 'eqiad', u'45278e02-7898-4aa6-a8eb-98c677c6b60c', [u'overpass-wiki'])
(u'math', 'eqiad', u'e9587207-1fa2-4faf-926b-d9efc41b176c', [u'drmf'])
(u'mwoffliner', 'eqiad', u'e9587207-1fa2-4faf-926b-d9efc41b176c', [u'mwoffliner3', u'mwoffliner2'])
(u'openstack', 'eqiad', u'f2ef20b4-0a51-4b0b-a4ea-2c7186fcb91c', [u'labs-bootstrapvz-stretch'])
(u'swift', 'eqiad', u'34964ef5-ac95-4e21-8f7d-ecf8e58978bb', [u'swift-stretch-ms-be02', u'swift-stretch-ms-be01', u'swift-stretch-ms-fe01'])
(u'tools', 'eqiad', u'e9587207-1fa2-4faf-926b-d9efc41b176c', [u'tools-checker-01', u'tools-webgrid-generic-1404', u'tools-webgrid-generic-1403', u'tools-webgrid-generic-1402', u'tools-webgrid-generic-1401', u'tools-webgrid-lighttpd-1410', u'tools-webgrid-lighttpd-1409', u'tools-webgrid-lighttpd-1408', u'tools-webgrid-lighttpd-1407', u'tools-webgrid-lighttpd-1406', u'tools-webgrid-lighttpd-1405', u'tools-webgrid-lighttpd-1404', u'tools-webgrid-lighttpd-1403', u'tools-webgrid-lighttpd-1402', u'tools-webgrid-lighttpd-1401', u'tools-checker-02', u'tools-exec-1410', u'tools-exec-1409', u'tools-exec-1408', u'tools-exec-1407', u'tools-exec-1406', u'tools-exec-1405', u'tools-exec-1404', u'tools-exec-1403', u'tools-exec-1402', u'tools-exec-1401', u'tools-bastion-02', u'tools-services-01', u'tools-services-02'])
(u'wikidumpparse', 'eqiad', u'e9587207-1fa2-4faf-926b-d9efc41b176c', [u'wigi'])
(u'wikistats', 'eqiad', u'34964ef5-ac95-4e21-8f7d-ecf8e58978bb', [u'wikistats-kraken'])

Event Timeline

Code
import collections

from keystoneclient.session import Session as KeystoneSession
from keystoneclient.auth.identity.v3 import Password as KeystonePassword
from keystoneclient.v3 import Client as KeystoneClient
from novaclient import client as novaclient
from glanceclient import client as glanceclient

def get_keystone_session(project):
    return KeystoneSession(auth=KeystonePassword(
        auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
        username="novaobserver",
        password='Fs6Dq2RtG8KwmM2Z', # public guest password
        project_name=project,
        user_domain_name='default',
        project_domain_name='default'
    ))

keystone_client = KeystoneClient(
    session=get_keystone_session('bastion'),
    endpoint="http://cloudcontrol1003.wikimedia.org:5000/v3",
    interface='public'
)

for project in keystone_client.projects.list():
    if project.name != 'admin':
        session = get_keystone_session(project.name)
        for region in ['eqiad', 'eqiad1-r']:
            nclient = novaclient.Client("2.0", session=session, region_name=region)
            image_instances = collections.defaultdict(list)
            for s in nclient.servers.list():
                image_instances[s.image['id']].append(s.name)
            gclient = glanceclient.Client(version=2, session=session, region_name=region)
            existing_images = {}
            for image in gclient.images.list():
                existing_images[image.id] = image.name
            for missing_image in set(image_instances.keys()) - set(existing_images.keys()):
                print(project.name, region, missing_image, image_instances[missing_image])

A couple of different things are happening here. One is that in many cases the image exists but is inaccessible to the project using it. I'm fixing those one by one.

The other issue is that gclient.images.list() returns owned images but doesn't return private images that are visible to the current project but not public or owned by that project. I don't know why, but the upcoming code addresses this, badly.

import collections

from keystoneclient.session import Session as KeystoneSession
from keystoneclient.auth.identity.v3 import Password as KeystonePassword
from keystoneclient.v3 import Client as KeystoneClient
from novaclient import client as novaclient
from glanceclient import client as glanceclient

def get_keystone_session(project):
    return KeystoneSession(auth=KeystonePassword(
        auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
        username="novaobserver",
        password='Fs6Dq2RtG8KwmM2Z', # public guest password
        project_name=project,
        user_domain_name='default',
        project_domain_name='default'
    ))

keystone_client = KeystoneClient(
    session=get_keystone_session('bastion'),
    endpoint="http://cloudcontrol1003.wikimedia.org:5000/v3",
    interface='public'
)

for project in keystone_client.projects.list():
    if project.name != 'admin':
        session = get_keystone_session(project.name)
        for region in ['eqiad', 'eqiad1-r']:
            missing_images=[]
            nclient = novaclient.Client("2.0", session=session, region_name=region)
            image_instances = collections.defaultdict(list)
            for s in nclient.servers.list():
                image_instances[s.image['id']].append(s.name)
            gclient = glanceclient.Client(version=2, session=session, region_name=region)
            existing_images = {}
            for image in image_instances:
                try:
                    image_obj = gclient.images.get(image)
                except:
                    missing_images.append(image)
            for missing_image in missing_images:
                print(project.name, region, missing_image, image_instances[missing_image])

$ python ./findDodgyInstances.py
(u'maps', 'eqiad', u'45278e02-7898-4aa6-a8eb-98c677c6b60c', [u'overpass-wiki'])
(u'openstack', 'eqiad', u'f2ef20b4-0a51-4b0b-a4ea-2c7186fcb91c', [u'labs-bootstrapvz-stretch'])
(u'swift', 'eqiad', u'34964ef5-ac95-4e21-8f7d-ecf8e58978bb', [u'swift-stretch-ms-be02', u'swift-stretch-ms-be01', u'swift-stretch-ms-fe01'])
(u'wikistats', 'eqiad', u'34964ef5-ac95-4e21-8f7d-ecf8e58978bb', [u'wikistats-kraken'])