Page MenuHomePhabricator
Paste P69432

create-large-item.py
ActivePublic

Authored by Ollie.Shotton_WMDE on Sep 27 2024, 12:36 PM.
Project Tags
None
Referenced Files
F57562209: create-large-item.py
Sep 27 2024, 12:36 PM
Subscribers
None
import os
import random
import string
import time
import requests
REST_BASE = 'https://wikidata.beta.wmflabs.org/w/rest.php'
BASE_URL = f'{REST_BASE}/wikibase/v0'
PREDICATE_PROPERTY_ID = 'P249598'
QUALIFIER_PROPERTY_ID = 'P216060'
REFERENCE_PROPERTY_ID = 'P230880'
headers = {
'User-Agent': 'WMDE WPP create large item script',
'Authorization': f'Bearer {os.environ["BEARER_TOKEN"]}'
}
def create_unique_string(prefix: str = '') -> str:
unique = ''.join(random.choices(string.ascii_letters, k=10))
return f'{prefix} {unique}'.strip()
def create_random_property_value_pair(property_id: str) -> dict:
return {
'property': {
'id': property_id,
},
'value': {
'type': 'value',
'content': create_unique_string('random string value'),
}
}
def create_random_statement() -> dict:
return {
**create_random_property_value_pair(PREDICATE_PROPERTY_ID),
'qualifiers': [create_random_property_value_pair(QUALIFIER_PROPERTY_ID) for _ in range(5)],
'references': [{'parts': [create_random_property_value_pair(REFERENCE_PROPERTY_ID) for _ in range(5)]} for _ in range(5)],
}
# create an item with a single statement
# body = {
# "item": {
# "labels": {"en": create_unique_string('large item')},
# 'statements': {PREDICATE_PROPERTY_ID: [create_random_statement()]},
# }
# }
# response = requests.request("POST", f"{BASE_URL}/entities/items", headers=headers, json=body)
#
# if response.status_code > 201:
# print('Error creating item')
# print('Status code:', response.status_code)
# print('Headers:', response.headers)
# print('Body:', response.text)
# exit()
#
# item_id = response.json()['id']
item_id = 'Q630399'
statements = {}
batch_size = 50
while batch_size > 0:
start = time.time()
body = {
'patch': [
{'op': 'add', 'path': f'/statements/{PREDICATE_PROPERTY_ID}/-', 'value': create_random_statement()}
for _ in range(batch_size)
]
}
response = requests.request("PATCH", f"{BASE_URL}/entities/items/{item_id}", headers=headers, json=body)
end = time.time()
if response.status_code == 400 and response.json()['code'] == 'resource-too-large':
batch_size = batch_size // 2
continue
# if response.status_code == 429:
# TODO: handle rate limiting?
if response.status_code != 200:
print('Error patching item')
print('Status code:', response.status_code)
print('Headers:', response.headers)
print('Body:', response.text)
exit()
statements = response.json()['statements'][PREDICATE_PROPERTY_ID]
print(f'Added {batch_size} statements to {item_id} in {end-start} seconds. Total statements: {len(statements)}')
print(f"created large item with {len(statements)} statements")