pccvtc.py
ActivePublic
Actions

Authored by • ema on Apr 27 2019, 12:12 PM.

Tags

None

Referenced Files

	F28808008: raw.txt
	Apr 27 2019, 12:12 PM

Subscribers

None

	#!/usr/bin/env python3

	import os
	import re
	import sys

	import requests

	DC = ('eqiad', 'codfw', 'esams', 'ulsfo', 'eqsin')
	CLUSTERS = ('text', 'upload')
	PATH_RE = re.compile('^(/etc/varnish\|/usr/share/varnish/)')
	COMPILER_RE = re.compile('.*(https://puppet-compiler.wmflabs.org/compiler[0-9]{4}/[0-9]+/)')
	PCC = '../../../../utils/pcc'

	def find_cluster(hostname):
	# eg: cp4021.ulsfo.wmnet -> DC[3] -> 'ulsfo'
	idx = int(hostname[2]) - 1
	dc = DC[idx]

	for cluster in CLUSTERS:
	url = 'https://config-master.wikimedia.org/pybal/{}/{}'.format(dc, cluster)
	r = requests.get(url)
	if hostname in r.text:
	return cluster

	raise Exception('Unknown cluster for {}'.format(hostname))

	def get_pcc_url(hostname, patch_id):
	cmd = ' '.join((PCC, patch_id, hostname))
	for line in os.popen(cmd).readlines():
	match = COMPILER_RE.match(line)
	if match:
	return match.group(1)

	raise Exception('Issues with get_pcc_url()')

	def dump_files(url, hostname):
	catalog_url = '{}/{}/change.{}.pson'.format(url, hostname, hostname)
	print('\tCatalog URL: {}'.format(catalog_url))

	catalog = requests.get(catalog_url).json()
	for resource in catalog['resources']:
	if resource['type'] != 'File':
	continue
	if PATH_RE.match(resource['title']) is None:
	continue
	if 'content' not in resource['parameters']:
	continue
	path = resource['title'].lstrip('/')
	os.makedirs(os.path.dirname(path), exist_ok=True)
	with open(path, 'wb') as f:
	print('\tCreating {}'.format(path))
	f.write(resource['parameters']['content'].encode('utf-8'))

	def main(hostname, patch_id='HEAD'):
	print('[*] running Puppet Catalog Compiler for change {}...'.format(patch_id))
	pcc_url = get_pcc_url(hostname, patch_id)
	print('\tPCC URL: {}'.format(pcc_url))

	print('[*] Dumping files...')
	dump_files(pcc_url, hostname)

	print('[*] Finding cluster...')
	cluster = find_cluster(hostname)
	print('\t{} is a cache_{} host'.format(cluster))

	print('[*] Running varnishtest...')
	cmd = 'sudo varnishtest {}/*.vtc'.format(cluster)
	print(cmd)

	if __name__ == '__main__':
	if len(sys.argv) != 3:
	print('Usage: {} hostname patch_id'.format(sys.argv[0]))
	sys.exit(1)

	main(sys.argv[1], sys.argv[2])

Event Timeline

• ema created this paste.Apr 27 2019, 12:12 PM

pccvtc.pyActivePublicActions

Event Timeline

pccvtc.py
ActivePublic
Actions