deepcat investigation of internal query limits
ActivePublic
Actions

Authored by EBernhardson on Aug 30 2024, 7:53 PM.

Project Tags

None

Referenced Files

	F57360493: deepcat investigation of internal query limits
	Aug 30 2024, 7:53 PM

Subscribers

None

	from pprint import pprint
	import requests
	import sys

	API_ENDPOINT = 'https://commons.wikimedia.org/w/api.php'
	BASE_API_QUERY = {
	'action': 'query',
	'format': 'json',
	'formatversion': '2',
	'list': 'allpages',
	'apnamespace': '14',
	'aplimit': '500',
	}

	def build_es_query(results):
	prefix = 'Category:'
	return {
	"size": 1,
	"track_total_hits": True,
	"_source": ["title"],
	"query": {
	"bool": {
	"filter": [
	{
	"bool": {
	"minimum_should_match": 1,
	"should": [
	{
	"bool": {
	"minimum_should_match": 1,
	"should": [
	{
	"match": {
	"category.lowercase_keyword": {
	"query": page['title'][len(prefix) + 1:]
	}
	}
	}
	for page in pages
	]
	}
	}
	for pages in results
	]
	}
	}
	]
	}
	}
	}

	results = []
	api_query = BASE_API_QUERY
	metrics = []
	for i in range(200):
	print(f"Requesting: {api_query}", file=sys.stdout, flush=True)
	res = requests.get(API_ENDPOINT, params=api_query)
	res.raise_for_status()
	api_query = dict(BASE_API_QUERY, **res.json()['continue'])
	results.append(res.json()['query']['allpages'])

	print(f"Total categories: {sum(len(pages) for pages in results)}", file=sys.stdout, flush=True)
	raw_res = requests.get('https://search.svc.eqiad.wmnet:9243/commonswiki_file/_search', json=build_es_query(results))
	raw_res.raise_for_status()
	res = raw_res.json()
	metrics.append({
	"categories": sum(len(pages) for pages in results),
	"hits": res['hits']['total']['value'],
	"took": res['took']
	})
	pprint(metrics[-1])

	print('\|categories\|hits\|took\|')
	print('\|--\|--\|--\|')
	for data in metrics:
	print(f"\|{data['categories']}\|{data['hits']}\|{data['took']}\|")

Event Timeline