Page MenuHomePhabricator
Paste P7450

MediaWiki first_upper vs Python
ActivePublic
Actions

Authored by Dalba on Aug 11 2018, 2:54 PM.
Project Tags
None
Referenced Files
F24843119: MediaWiki first_upper vs Python
Aug 12 2018, 8:58 AM
F24841969: MediaWiki first_upper vs Python
Aug 12 2018, 8:27 AM
F24835946: mediawiki first_upper
Aug 12 2018, 6:14 AM
F24823702: mediawiki first_upper
Aug 11 2018, 9:52 PM
F24818697: mediawiki first_upper
Aug 11 2018, 6:15 PM
F24811094: mediawiki first_upper
Aug 11 2018, 2:54 PM
Subscribers
None
# coding: utf-8
"""Run this module multiple times using different python versions.
Note: I seems that running under the latest version of Python (3.7) gives a
superse of the older version and should be enough. But I have not tested this
completely.
"""
from __future__ import unicode_literals, absolute_import
from sys import maxunicode, version_info
from re import findall
from json import dump, load, dumps
from threading import Thread, Lock
from scripts.maintenance.wikimedia_sites import families_list
from pywikibot.family import Family
from pywikibot import Site
from pywikibot.comms.http import session
if version_info[0] == 2:
from io import open
from Queue import Queue
chr = unichr
else:
from queue import Queue
NUMBER_OF_THREADS = 26
FILEPATH = '/data/project/dbbot/www/static/firstup_excepts.json'
def chars_uppers_wikilinks():
n = 0
chars = []
uppers = []
wikilinks = ''
for i in range(0, maxunicode + 1):
c = chr(i)
uc = c.upper()
if uc != c:
n += 1
chars.append(c)
uppers.append(uc)
# MediaWiki is first-letter case
wikilinks += '[[MediaWiki:' + c + ']]\n'
return chars, uppers, wikilinks
def process_site(fam_name, site_code):
j = session.post(
'https://{site_code}.{fam_name}.org/w/api.php?'
'action=parse&contentmodel=wikitext&prop=text'
'&format=json&utf8'.format(**locals()),
data={'text': wikilinks},
timeout=10,
).json()
pased_text = j["parse"]["text"]['*']
titles = findall(r'title="[^:]*:(.)', pased_text)
site_excepts = {}
for i, original_char in enumerate(chars):
title_char = titles[i]
if uppers[i] != title_char:
site_excepts[original_char] = title_char
return site_excepts
def threads_target(q):
while True:
try:
fam, code = q.get()
except TypeError: # non-iterable NoneType object
break
site_excepts = process_site(fam, code)
families_excepts[fam].setdefault(code, {}).update(site_excepts)
q.task_done()
def spawn_threads(q):
threads = []
for i in range(NUMBER_OF_THREADS):
t = Thread(target=threads_target, args=(q,))
t.start()
threads.append(t)
return threads
def stop_threads(q, threads):
for i in range(NUMBER_OF_THREADS):
q.put(None)
for t in threads:
t.join()
def main():
q = Queue()
threads = spawn_threads(q)
for fam_name in families_list:
family = Family.load(fam_name)
families_excepts.setdefault(fam_name, {})
for site_code in family.languages_by_size:
site = Site(site_code, family)
if site.namespaces[8].case != 'first-letter':
raise ValueError('MW namespace case is not first-letter')
fam_code = (fam_name, site_code)
if fam_code in {
('wikisource', 'www'),
('wikisource', 'mul'),
('wikiversity', 'test'),
}:
continue # the API of these codes does not respond as expected
q.put(fam_code)
# block until all tasks are done
q.join()
stop_threads(q, threads)
def save_json(obj, path):
with open(path, 'w', encoding='utf8') as f:
try:
dump(obj, f)
except TypeError: # Python 2 TypeError: must be unicode, not str
f.write(unicode(dumps(obj)))
def load_json(path):
try:
with open(path, 'r', encoding='utf8') as f:
return load(f)
except (OSError, IOError):
print('File not found:', path)
return {}
if __name__ == '__main__':
chars, uppers, wikilinks = chars_uppers_wikilinks()
# save_json({'chars': chars, 'uppers': uppers, 'wikilinks': wikilinks}, 'user-temp-save.json')
# j = load_json('user-temp-save.json')
# chars, uppers, wikilinks = j['chars'], j['uppers'], j['wikilinks']
# families_excepts = load_json(FILEPATH)
# main()
# save_json(families_excepts, FILEPATH)
print(process_site('wiktionary', 'fr'))
Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits