Compare images from 2 thumbor instances
ActivePublic
Actions

Authored by jijiki on Feb 26 2019, 1:04 PM.

Tags

Referenced Files

	F28288819: raw.txt
	Feb 26 2019, 1:06 PM

	F28288817: raw.txt
	Feb 26 2019, 1:06 PM

	F28288792: raw.txt
	Feb 26 2019, 1:04 PM

Subscribers

• Gilles

jijiki

	#!/usr/bin/python
	''' Need to set RESULT_STORAGE = None (so not to save output to swift)'''

	import logging
	import sys
	import json
	import urllib
	import urllib2
	import tempfile
	import argparse
	from ssim import compute_ssim
	from PIL import Image

	FORMATTER = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")

	logger = logging.getLogger()
	logger.setLevel(logging.ERROR)
	console_handler = logging.StreamHandler(sys.stdout)
	console_handler.setFormatter(FORMATTER)
	logger.addHandler(console_handler)

	parser = argparse.ArgumentParser(description='Do a visual diff between a test host and production.')
	parser.add_argument('extension', metavar='ext', help='The specific file extension to test')

	args = parser.parse_args()

	random_batch_size = 500
	amount = 100
	thumbnail_batch_size = 10
	width = 150
	host1 = 'http://thumbor2001:8800/'
	host2 = 'http://thumbor2002:8800/'

	titles = []

	def get_random_titles():
	random_titles = []

	while len(random_titles) < amount:
	logger.debug('Asking for a new batch of %d random titles' % random_batch_size)
	response = urllib2.urlopen('https://commons.wikimedia.org/w/api.php?action=query&format=json&list=random&rnnamespace=6&rnlimit=%d' % random_batch_size)

	output = response.read()

	results = json.loads(output)

	for result in results['query']['random']:
	title = result['title']
	if title.endswith(args.extension):
	random_titles.append(urllib.quote(title.encode('utf-8')))

	random_titles = random_titles[amount:]

	return random_titles


	titles = [
	'File:086e53_f599b34e9fc646349cccd5987b6176cd.webp',
	'File:1b0c67_42799771475042c2b603d7b665b8d3db_mv2.webp',
	'File:1f3c0f_07d96ede1163492fa633916369e2f67e.webp',
	'File:2018_Capitol_Hill_Film_Classic_Red_Carpet.webp',
	'File:38edf3_e24896398e484f7bb4f5a6184f612c6d.webp',
	'File:3f4c10_e67b0c99c6c940f4b11f81c64625a19f-mv2.webp',
	'File:4b3139b30cb85e0d3a03a94ab738b7b7_(1).webp',
	'File:4f95eb_a8a2f63a15ed4e51886f4f8afa5729f3_mv2.webp',
	'File:530b3d57ed265article_2282_1.webp',
	'File:5A-LAR.webp',
	'File:%22What_a_Brilliant_Idea!%22_Barnstar.xcf',
	'File:%27s-Hertogenbosch.xcf',
	'File:(2)Sakramentsh%C3%A4uschen_St._Peter_und_Paul_G%C3%B6%C3%9Flingen.xcf',
	'File:(cs)_WM_CEE_Poster_A3_300_dpi.xcf',
	'File:12o.xcf',
	'File:15-09-11_006_Quark_Expeditions_ship,_Sea_Adventurer_(Nassau_registry_IMO_7391422),_at_Griffin_Inlet,_Beechey_Island,_Nunavut,_Canada.xcf',
	'File:151027_Varga_Csaba_70_megeml%C3%A9kez%C3%A9s_02_Buglya_S%C3%A1ndor.xcf',
	'File:151027_Varga_Csaba_70_megeml%C3%A9kez%C3%A9s_07_M%C3%A9sz%C3%A1ros_G%C3%A9za.xcf',
	'File:17th_century_Spanish_routes_to_Pet%C3%A9n.xcf',
	'File:1_%D0%91%D0%B5%D0%B7_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B8.xcf',
	'File:0.25_watts_through-hole_resistor.stl',
	'File:1-msr-agrippa-postumus-5.stl',
	'File:10-msr-bust-in-armor-repaired-repaired-cut-2.stl',
	'File:10_cubic_centimetres_volume_calibration_part_(20x50_mm).stl',
	'File:11-msr-unknown-bust-10-repaired.stl',
	'File:12-msr-unknown-bust-2-10.stl',
	'File:13-msr-unknown-bust-3-10.stl',
	'File:14-msr-unknown-bust-of-a-woman-10.stl',
	'File:15-msr-bust-of-a-young-man-10.stl',
	'File:18-msr-bust-of-a-woman-2-10.stl'
	]

	while len(titles):
	batch = titles[:10]
	titles = titles[10:]

	thumb_api_request = 'https://commons.wikimedia.org/w/api.php?action=query&format=json&prop=imageinfo&titles=%s&iiurlwidth=%d&iiprop=url' % ('\|'.join(batch), width)

	response = urllib2.urlopen(thumb_api_request)

	output = response.read()

	results = json.loads(output)

	for page in results['query']['pages']:
	original_url = results['query']['pages'][page]['imageinfo'][0]['thumburl']

	logger.debug('Processing %s' % original_url)

	# Strip https://upload.wikimedia.org/
	url = original_url[len('https://upload.wikimedia.org/'):]

	fh, path1 = tempfile.mkstemp()
	thumb1 = urllib.urlretrieve(host1 + url, path1)

	fh, path2 = tempfile.mkstemp()
	thumb2 = urllib.urlretrieve(host1 + url, path2)

	try:
	image1 = Image.open(path1)
	image2 = Image.open(path2)

	ssim = compute_ssim(image1, image2)

	if ssim < 1:
	logger.error('SSIM of %d for %s' % (ssim, original_url))
	except IOError:
	logger.error('Pillow could not process this file: %s' % original_url)

Event Timeline

jijiki created this paste.Feb 26 2019, 1:04 PM

jijiki edited the content of this paste. (Show Details)

jijiki edited the content of this paste. (Show Details)Feb 26 2019, 1:06 PM

Compare images from 2 thumbor instances ActivePublicActions

Event Timeline

Compare images from 2 thumbor instances
ActivePublic
Actions