Page MenuHomePhabricator
Paste P3854

scrape deployment calendar for patches to swat
ActivePublic
Actions

Authored by • mmodell on Aug 19 2016, 6:38 AM.
from bs4 import BeautifulSoup
from datetime import datetime
import dateutil.parser
from dateutil.tz import tzlocal
from dateutil.relativedelta import relativedelta
import re
import requests
import scap.cli as cli
@cli.command('swat', help='Mediawiki SWAT deployment helper.')
class Swat(cli.Application):
@cli.argument('--cherry-pick', action='store_true',
help='Cherry-pick the patches instead of merging.')
@cli.argument('-b', '--branch', nargs="+",
help='One or more branches to merge into. Default: current branch.')
@cli.argument('--base', nargs=1,
help='Rebase the patch against this commit before merging.')
@cli.argument('--changeid', nargs="+",
help='The ChangeId of a patch to merge.')
@cli.argument('--start', action='store_true',
help='Enqueue patches from the deployment calendar and begin SWAT '
+ 'deployment.')
def main(self, *extra_args):
if self.arguments.start == True:
self.scrape_deployment_calendar()
def scrape_deployment_calendar(self):
gerrit_link_re = re.compile('https://gerrit.wikimedia.org')
def match_gerrit_link(tag):
return (
tag.name == 'a'
and tag.has_attr('href')
and gerrit_link_re.match(tag['href']))
NOW = datetime.now(tzlocal())
r = requests.get('https://wikitech.wikimedia.org/wiki/Deployments')
soup = BeautifulSoup( r.text, 'lxml' )
# find all swat deploy windows on the deployment calendar
for tag in soup.find_all(title="SWAT deploys"):
# look up 3 levels to find the TR tag
row = tag.parent.parent.parent
if row.name != 'tr':
continue
# get the timestamp from the TR tag's id attribute
datestring = row['id'].rsplit('-', 1)[1] + "Z"
# parse the date string
window_start = dateutil.parser.parse(datestring)
window_end = window_start + relativedelta(hours=+1)
# if the current time is not within this window, continue to next
if NOW <= window_start or NOW > window_end:
continue
# get all the names for developers with patches to deploy
people = row.find_all(class_='ircnick-container')
if len(people):
print("---------------------------------------------------")
print("from %s to %s" % (window_start, window_end))
for person in people:
name = person.text
cell = person.find_parent('td')
if person.parent.name == 'p':
# All but the first developers' names are nested inside a <p>
r = person.parent
else:
r = person
# get the list immediately following the developer name
nextTag = r.find_next_sibling('ul')
if not nextTag:
continue
# find all gerrit links in this list
links = nextTag.find_all(match_gerrit_link)
if not links:
continue
print("")
print(name)
print("------ ------------------------------------------")
for link in links:
print("%s - %s" % (link.text, link['href']))
Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits