# -*- coding: utf-8 -*-
"""
  cache_age
  ~~~~~~~~~
  Retrieve random pages from random Wikimedia projects and scrape
  their cache age. Then write the ages to a file on disk.

"""
import datetime
import random
import re

import requests


NUM_PAGES = 10000

def get_sites():
    params = {'action': 'sitematrix', 'format': 'json'}
    r = requests.get('https://meta.wikimedia.org/w/api.php', params)
    site_matrix = r.json()['sitematrix'].values()
    projects = (p for p in site_matrix if isinstance(p, dict))
    sites = []
    for project in projects:
        for site in project.get('site', ()):
            sites.append(site['url'])
    return sites

sites = get_sites()

ages = []
while len(ages) < NUM_PAGES:
    site = random.choice(sites)
    r = requests.get(site + '/wiki/Special:Random')
    m = re.search(r'and timestamp (\d+) and revision', r.text)
    if not m: continue
    parsed = datetime.datetime.strptime(m.group(1), '%Y%m%d%H%M%S')
    delta = datetime.datetime.utcnow() - parsed
    age = delta.total_seconds()
    ages.append(age)


with open('html_ages.csv', 'w') as f:
    for age in ages:
        print(age, file=f)