Page MenuHomePhabricator
Paste P4805

Map tool users to precise tools
ActivePublic

Authored by madhuvishy on Jan 25 2017, 9:12 AM.
Tags
None
Referenced Files
F5363296: Map tool users to precise tools
Jan 26 2017, 4:37 PM
F5363290: Map tool users to precise tools
Jan 26 2017, 4:33 PM
F5357747: Map tool users to precise tools
Jan 25 2017, 7:44 PM
F5357721: Map tool users to precise tools
Jan 25 2017, 7:33 PM
F5354663: Map tool users to precise tools
Jan 25 2017, 9:13 AM
F5354662: Map tool users to precise tools
Jan 25 2017, 9:13 AM
F5354661: Map tool users to precise tools
Jan 25 2017, 9:12 AM
Tokens
"Party Time" token, awarded by bd808.
#!/usr/bin/python3
# Copyright 2016 Madhumitha Viswanathan mviswanathan@wikimedia.org
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
#
# Maps tool users to precise tools
# Run like tail -500000 /data/project/.system/accounting | ./precise-owners.py
# from tools bastions
#
import collections
import datetime
import fileinput
import http.client
import json
import ldap3
import operator
import yaml
def ldap_conn(config):
"""
Return a ldap connection
Return value can be used as a context manager
"""
servers = ldap3.ServerPool([
ldap3.Server(host)
for host in config['servers']
], ldap3.POOLING_STRATEGY_ROUND_ROBIN, active=True, exhaust=True)
return ldap3.Connection(servers, read_only=True,
user=config['user'],
auto_bind=True,
password=config['password'])
def uid_from_dn(dn):
keys = dn.split(',')
uid_key = keys[0]
uid = uid_key.split('=')[1]
return uid
def tools_members(config, tools):
"""
Return a dict that has members of a tool associated with each tool
Ex:
{'tools.musikbot': ['musikanimal'],
'tools.ifttt': ['slaporte', 'mahmoud', 'madhuvishy', 'ori']}
"""
tool_to_members = collections.defaultdict(list)
with ldap_conn(config) as conn:
for tool in tools:
conn.search(
'ou=servicegroups,dc=wikimedia,dc=org',
'(cn={})'.format(tool),
ldap3.SEARCH_SCOPE_WHOLE_SUBTREE,
attributes=['member', 'cn'],
time_limit=5
)
for resp in conn.response:
attributes = resp.get('attributes')
members = attributes['member'] if attributes.get('member') else []
tool_to_members[tool].extend([uid_from_dn(member) for member in members])
return tool_to_members
def members_tools(tool_to_members):
all_members = set().union(*tool_to_members.values())
member_to_tools = collections.defaultdict(list)
for tool, members in tool_to_members.items():
for m in members:
if tool not in member_to_tools[m]:
member_to_tools[m].append(tool)
return member_to_tools
def is_precise_host(hostname):
if hostname[-4:].startswith('12'):
return True
def grid_precise_tools():
all_precise_tools = []
conn = http.client.HTTPConnection('tools.wmflabs.org')
conn.request("GET", "/gridengine-status",
headers={"User-Agent": "Precise tools finder|labs-admin@lists.wikimedia.org"})
res = conn.getresponse().read().decode('utf-8')
if res:
grid_info = json.loads(res)["data"]["attributes"]
for hostname, info in grid_info.items():
if is_precise_host(hostname):
if info["jobs"]:
all_precise_tools.extend([job["job_owner"] for job in info["jobs"].values()])
return all_precise_tools
def accounting_tools():
DAYS=7
FIELD_NAMES = [
'qname', 'hostname', 'group', 'owner', 'job_name', 'job_number', 'account',
'priority', 'submission_time', 'start_time', 'end_time', 'failed',
'exit_status', 'ru_wallclock', 'ru_utime', 'ru_stime', 'ru_maxrss',
'ru_ixrss', 'ru_ismrss', 'ru_idrss', 'ru_isrss', 'ru_minflt', 'ru_majflt',
'ru_nswap', 'ru_inblock', 'ru_oublock', 'ru_msgsnd', 'ru_msgrcv',
'ru_nsignals', 'ru_nvcsw', 'ru_nivcsw', 'project', 'department',
'granted_pe', 'slots', 'task_number', 'cpu', 'mem', 'io', 'category',
'iow', 'pe_taskid', 'maxvemem', 'arid', 'ar_submission_time',
]
cutoff = (datetime.datetime.now() - datetime.timedelta(days=DAYS)).timestamp()
precise_tools = []
for line in fileinput.input():
parts = line.split(':')
job = dict(zip(FIELD_NAMES, parts))
if int(job['end_time']) < cutoff:
continue
if 'release=precise' in job['category'] and job['owner'] not in precise_tools:
precise_tools.append(job['owner'])
return precise_tools
with open('/etc/ldap.yaml') as f:
config = yaml.safe_load(f)
tool_to_members = tools_members(config, accounting_tools() + grid_precise_tools())
mt = members_tools(tool_to_members)
print json.dumps(mt, sort_keys=True, indent=4, separators=(',', ': '))

Event Timeline

madhuvishy edited the content of this paste. (Show Details)
madhuvishy edited the content of this paste. (Show Details)
This comment was removed by madhuvishy.

Members to precise tools mapping (json)

{
    "a930913": [
        "tools.bbc-tv-cite",
        "tools.cluestuff",
        "tools.defconbot"
    ],
    "addshore": [
        "tools.massaction",
        "tools.cgstat"
    ],
    "afeder": [
        "tools.edinc"
    ],
    "agx": [
        "tools.ptwikis"
    ],
    "ainali": [
        "tools.slumpartikel"
    ],
    "aka": [
        "tools.inactiveadmins",
        "tools.listpages",
        "tools.revertstat",
        "tools.germancontributioncounts"
    ],
    "akoopal": [
        "tools.locator",
        "tools.nlwikibots",
        "tools.erwin85"
    ],
    "alchimista": [
        "tools.cobain",
        "tools.ptwikis"
    ],
    "alessio": [
        "tools.lists"
    ],
    "alexxw": [
        "tools.wahldiagramm",
        "tools.anno"
    ],
    "alexz": [
        "tools.popularpages",
        "tools.amdb",
        "tools.zbot",
        "tools.reftoolbar",
        "tools.geophotoreq"
    ],
    "alphos": [
        "tools.erwin85"
    ],
    "andrew": [
        "tools.toolschecker"
    ],
    "andyrussg": [
        "tools.coursestats"
    ],
    "anorange0409": [
        "tools.yichengtry"
    ],
    "anskar-cawiki": [
        "tools.cobain"
    ],
    "aperson": [
        "tools.badges"
    ],
    "apper": [
        "tools.wikihistory",
        "tools.spellcheck",
        "tools.ipp"
    ],
    "arashpt": [
        "tools.ptbot"
    ],
    "arnaugir": [
        "tools.cobain"
    ],
    "ashig": [
        "tools.wikidata-nolabels"
    ],
    "atsirlin": [
        "tools.wikivoyage"
    ],
    "aude": [
        "tools.audetools"
    ],
    "avicennasis": [
        "tools.avicbot"
    ],
    "bawolff": [
        "tools.bawolff"
    ],
    "beetstra": [
        "tools.chemobot",
        "tools.unblockbot"
    ],
    "beta16": [
        "tools.betabot"
    ],
    "betacommand": [
        "tools.betacommand-dev"
    ],
    "blahma": [
        "tools.cssk",
        "tools.blahma"
    ],
    "castor": [
        "tools.cobain"
    ],
    "cbm": [
        "tools.veblenbot"
    ],
    "cdpark": [
        "tools.chobot"
    ],
    "chm": [
        "tools.slumpartikel"
    ],
    "coet": [
        "tools.cobain"
    ],
    "cyberpower678": [
        "tools.wikihistory",
        "tools.xtools",
        "tools.cyberbot",
        "tools.supercount",
        "tools.erwin85"
    ],
    "dalba": [
        "tools.yadkard"
    ],
    "damian": [
        "tools.cluestuff",
        "tools.cluebot3"
    ],
    "daniel": [
        "tools.commonscategorycount",
        "tools.potd-feed"
    ],
    "danilo": [
        "tools.ptwikis"
    ],
    "danmichaelo": [
        "tools.stemmeberettigelse",
        "tools.arkivbot",
        "tools.sdbot",
        "tools.catmonitor",
        "tools.matvaretabellen"
    ],
    "dapete": [
        "tools.dewikinews-rss",
        "tools.imagemapedit",
        "tools.random-featured"
    ],
    "darkdadaah": [
        "tools.wikt-mwtest",
        "tools.dicompte",
        "tools.wiktioutils"
    ],
    "dartar": [
        "tools.cite-o-meter"
    ],
    "deba": [
        "tools.analytalks"
    ],
    "deltaquad": [
        "tools.hat-collector"
    ],
    "dereckson": [
        "tools.translate",
        "tools.wikidata-nolabels"
    ],
    "diego": [
        "tools.ptwikis"
    ],
    "dm": [
        "tools.dibot"
    ],
    "drtrigon": [
        "tools.drtrigonbot"
    ],
    "dschwen": [
        "tools.dschwenbot",
        "tools.osm"
    ],
    "dungodung": [
        "tools.srwiki"
    ],
    "dz": [
        "tools.heimdall"
    ],
    "ebrahim": [
        "tools.fawikiauto"
    ],
    "eccenux": [
        "tools.authors",
        "tools.dna"
    ],
    "edoderoo": [
        "tools.nlwikibots"
    ],
    "elee": [
        "tools.wikihistory"
    ],
    "eranroz": [
        "tools.hewiki-tools",
        "tools.eranbot"
    ],
    "euku": [
        "tools.pb"
    ],
    "fale": [
        "tools.lists"
    ],
    "fnde": [
        "tools.drtrigonbot"
    ],
    "fredddie": [
        "tools.usrd-tools"
    ],
    "gerardduenas": [
        "tools.cobain"
    ],
    "gifti": [
        "tools.giftbot",
        "tools.drtrigonbot",
        "tools.catscan3"
    ],
    "gpaumier": [
        "tools.mrmetadata"
    ],
    "gryllida": [
        "tools.gpy"
    ],
    "gwiki": [
        "tools.yichengtry"
    ],
    "halfak": [
        "tools.suggestbot"
    ],
    "happy5214": [
        "tools.usrd-tools"
    ],
    "hartman": [
        "tools.locator",
        "tools.erwin85"
    ],
    "he7d3r": [
        "tools.ptwikis"
    ],
    "hedonil": [
        "tools.tools-info"
    ],
    "henriquecrang": [
        "tools.ptwikis"
    ],
    "hercule": [
        "tools.herculebot"
    ],
    "hoo": [
        "tools.hoo",
        "tools.tree-of-life"
    ],
    "hprmedina": [
        "tools.grillitus"
    ],
    "hroest": [
        "tools.hroest"
    ],
    "hydriz": [
        "tools.simplewikt",
        "tools.simple"
    ],
    "incola": [
        "tools.incolabot",
        "tools.personabot",
        "tools.lists"
    ],
    "inkowik": [
        "tools.inkowik"
    ],
    "ireas": [
        "tools.stimmberechtigung",
        "tools.pb"
    ],
    "jackpotte": [
        "tools.jackbot",
        "tools.wikt-mwtest"
    ],
    "jarry1250": [
        "tools.wmukevents",
        "tools.templatecount",
        "tools.svgtranslate"
    ],
    "jason": [
        "tools.dplbot"
    ],
    "jcreus": [
        "tools.cobain"
    ],
    "jdlrobson": [
        "tools.styleguide"
    ],
    "jeanfred": [
        "tools.catbot",
        "tools.wm-metrics"
    ],
    "jem": [
        "tools.spellcheck"
    ],
    "jkroll": [
        "tools.commonscategorycount",
        "tools.cgstat"
    ],
    "jmo": [
        "tools.ftl"
    ],
    "johang": [
        "tools.wikitrends"
    ],
    "juanman": [
        "tools.plantel2wiki"
    ],
    "jurgennl": [
        "tools.jurgennl"
    ],
    "jzerebecki": [
        "tools.tree-of-life"
    ],
    "kaldari": [
        "tools.popularpages",
        "tools.eranbot",
        "tools.reftoolbar"
    ],
    "kangaroopower": [
        "tools.igloo"
    ],
    "kekaadrenalin": [
        "tools.citing-bot"
    ],
    "kevinpayravi": [
        "tools.traffic-grapher"
    ],
    "kjschiroo": [
        "tools.suggestbot"
    ],
    "knissen": [
        "tools.commonscategorycount"
    ],
    "kolossos": [
        "tools.zoomable-images",
        "tools.checkpersondata",
        "tools.catscan-pop",
        "tools.query2map",
        "tools.wiwosm",
        "tools.osm-add-tags",
        "tools.osm"
    ],
    "krd": [
        "tools.krdbot"
    ],
    "krlsca": [
        "tools.cobain"
    ],
    "ladsgroup": [
        "tools.mrmetadata",
        "tools.dexbot",
        "tools.fawikiauto"
    ],
    "lbenedix": [
        "tools.lbenedix"
    ],
    "legoktm": [
        "tools.globalprefs",
        "tools.cluestuff",
        "tools.legobot"
    ],
    "lena": [
        "tools.wm-metrics"
    ],
    "lestaty": [
        "tools.ptwikis"
    ],
    "liangent": [
        "tools.liangent-django",
        "tools.liangent-php"
    ],
    "lixxx235": [
        "tools.badges",
        "tools.hat-collector"
    ],
    "lokal-profil": [
        "tools.slumpartikel"
    ],
    "lucas559": [
        "tools.eranbot"
    ],
    "lucie": [
        "tools.tree-of-life"
    ],
    "madutgn": [
        "tools.cobain"
    ],
    "magioladitis": [
        "tools.awb"
    ],
    "magnus": [
        "tools.erwin85"
    ],
    "mahmoud": [
        "tools.revisionstats",
        "tools.weeklypedia"
    ],
    "marc": [
        "tools.toolschecker",
        "tools.shortnames"
    ],
    "matanya": [
        "tools.hewiki-tools"
    ],
    "mathbot": [
        "tools.mathbot"
    ],
    "mattflaschen": [
        "tools.styleguide"
    ],
    "matthewrbowker": [
        "tools.xtools"
    ],
    "maurelio": [
        "tools.hat-collector"
    ],
    "mauro742": [
        "tools.itwp-deletions"
    ],
    "mavrikant": [
        "tools.mavrikant"
    ],
    "maximilianklein": [
        "tools.cite-o-meter"
    ],
    "mbch331": [
        "tools.ytcleaner"
    ],
    "mdann52": [
        "tools.shortnames"
    ],
    "mf-warburg": [
        "tools.hat-collector"
    ],
    "miguillen": [
        "tools.mg-bot"
    ],
    "mormegil": [
        "tools.serviceawards"
    ],
    "multichill": [
        "tools.locator",
        "tools.multichill",
        "tools.nlwikibots",
        "tools.wlmtrafo",
        "tools.family",
        "tools.wikidata-janitor",
        "tools.catbot",
        "tools.railways"
    ],
    "musikanimal": [
        "tools.xtools",
        "tools.eranbot"
    ],
    "mxn": [
        "tools.tilde"
    ],
    "myst": [
        "tools.coord",
        "tools.cil2",
        "tools.duplicate-coords"
    ],
    "mzmcbride": [
        "tools.erwin85",
        "tools.tsreports"
    ],
    "nemobis": [
        "tools.erwin85"
    ],
    "nettrom": [
        "tools.suggestbot"
    ],
    "niharika29": [
        "tools.eranbot"
    ],
    "nknudsen": [
        "tools.cobain"
    ],
    "nn1l2": [
        "tools.nn1l2bot"
    ],
    "nullzero": [
        "tools.nullzerobot"
    ],
    "odi": [
        "tools.family"
    ],
    "papuass": [
        "tools.contributions-summary"
    ],
    "para": [
        "tools.para"
    ],
    "pasleim": [
        "tools.yichengtry",
        "tools.pltools"
    ],
    "peter17": [
        "tools.dioceses",
        "tools.powow"
    ],
    "petrb": [
        "tools.wm-bot"
    ],
    "phe": [
        "tools.phetools"
    ],
    "pierreselim": [
        "tools.wm-metrics"
    ],
    "pirsquared": [
        "tools.usergraph",
        "tools.erwin85"
    ],
    "pleclown": [
        "tools.ptools"
    ],
    "poulpy": [
        "tools.translate"
    ],
    "profoss": [
        "tools.arkivbot",
        "tools.catmonitor"
    ],
    "prolineserver": [
        "tools.slumpartikel"
    ],
    "prtksxna": [
        "tools.styleguide"
    ],
    "psychoslave": [
        "tools.wikt-mwtest"
    ],
    "ragesoss": [
        "tools.coursestats"
    ],
    "raylton-sousa": [
        "tools.ptwikis"
    ],
    "reedy": [
        "tools.awb"
    ],
    "retsam": [
        "tools.wiwosm",
        "tools.osm"
    ],
    "revi": [
        "tools.tc-rc"
    ],
    "reza": [
        "tools.fawikiauto"
    ],
    "rhaworth": [
        "tools.os"
    ],
    "richs": [
        "tools.cluebot3"
    ],
    "rillke": [
        "tools.commonsarchive",
        "tools.catbot"
    ],
    "rluts": [
        "tools.wle",
        "tools.lrbot"
    ],
    "rodrigopadula": [
        "tools.ptwikis"
    ],
    "rotpunkt": [
        "tools.personabot"
    ],
    "rschen7754": [
        "tools.usrd-tools",
        "tools.hat-collector"
    ],
    "ruhrfisch": [
        "tools.veblenbot"
    ],
    "russblau": [
        "tools.russbot",
        "tools.dplbot"
    ],
    "samwilson": [
        "tools.eranbot"
    ],
    "sanyi4": [
        "tools.lonelylinks"
    ],
    "scottywong": [
        "tools.usersearch"
    ],
    "se4598": [
        "tools.erwin85"
    ],
    "seth": [
        "tools.camelbot",
        "tools.url-converter",
        "tools.searchsbl"
    ],
    "siebrand": [
        "tools.nlwikibots"
    ],
    "sigma": [
        "tools.wikihistory",
        "tools.usersearch"
    ],
    "silverspoon": [
        "tools.locator",
        "tools.nlwikibots"
    ],
    "simon04": [
        "tools.cats-php",
        "tools.wiwosm"
    ],
    "sitic": [
        "tools.asurabot"
    ],
    "sk": [
        "tools.checkpersondata"
    ],
    "slaporte": [
        "tools.revisionstats",
        "tools.weeklypedia"
    ],
    "smith609": [
        "tools.timescale"
    ],
    "snaevar-bot": [
        "tools.snaevar-bot"
    ],
    "sonet": [
        "tools.wikitrip",
        "tools.wikiwatchdog"
    ],
    "spage": [
        "tools.styleguide"
    ],
    "spider": [
        "tools.wlmtrafo"
    ],
    "steenth": [
        "tools.dawikitool"
    ],
    "steinsplitter": [
        "tools.commonsarchive",
        "tools.yifeibot"
    ],
    "stemd": [
        "tools.hrwiki"
    ],
    "stuem007": [
        "tools.suggestbot"
    ],
    "symmachus": [
        "tools.badges"
    ],
    "tanvir": [
        "tools.welcomebots-bn"
    ],
    "tarrow": [
        "tools.wikifactmine-api"
    ],
    "taruuno": [
        "tools.welcomebots-bn"
    ],
    "taxonbot": [
        "tools.giftbot"
    ],
    "tb": [
        "tools.tb-dev"
    ],
    "tcn7jm": [
        "tools.usrd-tools",
        "tools.tc-rc"
    ],
    "technical-13": [
        "tools.xtools"
    ],
    "tgritschacher": [
        "tools.commonscategorycount"
    ],
    "theopolisme": [
        "tools.theoslittlebot",
        "tools.wikipedia-contributor-locations"
    ],
    "tools.mavrikant": [
        "tools.mavrikant"
    ],
    "tools.spbot": [
        "tools.pb"
    ],
    "tools.tsreports-dev": [
        "tools.tsreports"
    ],
    "tools.veblenbot": [
        "tools.veblenbot"
    ],
    "tools.yifeibot": [
        "tools.commonsarchive"
    ],
    "torty3": [
        "tools.wikivoyage"
    ],
    "tpt": [
        "tools.phetools"
    ],
    "unapersona": [
        "tools.cobain"
    ],
    "v2": [
        "tools.vtwo"
    ],
    "valhallasw": [
        "tools.nlwikibots",
        "tools.tsreports"
    ],
    "vrandezo": [
        "tools.yichengtry"
    ],
    "whym": [
        "tools.whymbot"
    ],
    "wiegels": [
        "tools.jury",
        "tools.pb",
        "tools.icalendar"
    ],
    "wmde-fisch": [
        "tools.commonscategorycount",
        "tools.cgstat"
    ],
    "yuvipanda": [
        "tools.toolschecker",
        "tools.test-lighttpd-precise"
    ],
    "zache-tool": [
        "tools.fiwiki-tools"
    ],
    "zhuyifei1999": [
        "tools.commonsarchive",
        "tools.wikivoyage",
        "tools.yifeibot"
    ]
}

Tool to members mapping

{
    "tools.amdb": [
        "alexz"
    ],
    "tools.analytalks": [
        "deba"
    ],
    "tools.anno": [
        "alexxw"
    ],
    "tools.arkivbot": [
        "danmichaelo",
        "profoss"
    ],
    "tools.asurabot": [
        "sitic",
        "sitic"
    ],
    "tools.audetools": [
        "aude"
    ],
    "tools.authors": [
        "eccenux"
    ],
    "tools.avicbot": [
        "avicennasis"
    ],
    "tools.awb": [
        "magioladitis",
        "reedy"
    ],
    "tools.badges": [
        "aperson",
        "lixxx235",
        "symmachus"
    ],
    "tools.bawolff": [
        "bawolff"
    ],
    "tools.bbc-tv-cite": [
        "a930913"
    ],
    "tools.betabot": [
        "beta16"
    ],
    "tools.betacommand-dev": [
        "betacommand"
    ],
    "tools.blahma": [
        "blahma"
    ],
    "tools.camelbot": [
        "seth",
        "seth"
    ],
    "tools.catbot": [
        "multichill",
        "jeanfred",
        "rillke"
    ],
    "tools.catmonitor": [
        "danmichaelo",
        "profoss"
    ],
    "tools.cats-php": [
        "simon04"
    ],
    "tools.catscan-pop": [
        "kolossos"
    ],
    "tools.catscan3": [
        "gifti"
    ],
    "tools.cgstat": [
        "addshore",
        "jkroll",
        "wmde-fisch",
        "addshore",
        "jkroll",
        "wmde-fisch"
    ],
    "tools.checkpersondata": [
        "kolossos",
        "sk"
    ],
    "tools.chemobot": [
        "beetstra"
    ],
    "tools.chobot": [
        "cdpark"
    ],
    "tools.cil2": [
        "myst"
    ],
    "tools.cite-o-meter": [
        "maximilianklein",
        "dartar"
    ],
    "tools.citing-bot": [
        "kekaadrenalin"
    ],
    "tools.cluebot3": [
        "damian",
        "richs"
    ],
    "tools.cluestuff": [
        "legoktm",
        "damian",
        "a930913"
    ],
    "tools.cobain": [
        "alchimista",
        "coet",
        "madutgn",
        "arnaugir",
        "jcreus",
        "anskar-cawiki",
        "nknudsen",
        "castor",
        "unapersona",
        "gerardduenas",
        "krlsca"
    ],
    "tools.commonsarchive": [
        "zhuyifei1999",
        "steinsplitter",
        "rillke",
        "tools.yifeibot"
    ],
    "tools.commonscategorycount": [
        "jkroll",
        "daniel",
        "knissen",
        "tgritschacher",
        "wmde-fisch"
    ],
    "tools.contributions-summary": [
        "papuass"
    ],
    "tools.coord": [
        "myst"
    ],
    "tools.coursestats": [
        "andyrussg",
        "ragesoss"
    ],
    "tools.cssk": [
        "blahma"
    ],
    "tools.cyberbot": [
        "cyberpower678"
    ],
    "tools.dawikitool": [
        "steenth"
    ],
    "tools.defconbot": [
        "a930913",
        "a930913"
    ],
    "tools.dewikinews-rss": [
        "dapete",
        "dapete"
    ],
    "tools.dexbot": [
        "ladsgroup"
    ],
    "tools.dibot": [
        "dm"
    ],
    "tools.dicompte": [
        "darkdadaah"
    ],
    "tools.dioceses": [
        "peter17"
    ],
    "tools.dna": [
        "eccenux"
    ],
    "tools.dplbot": [
        "russblau",
        "jason"
    ],
    "tools.drtrigonbot": [
        "drtrigon",
        "fnde",
        "gifti"
    ],
    "tools.dschwenbot": [
        "dschwen"
    ],
    "tools.duplicate-coords": [
        "myst"
    ],
    "tools.edinc": [
        "afeder"
    ],
    "tools.eranbot": [
        "eranroz",
        "kaldari",
        "lucas559",
        "musikanimal",
        "niharika29",
        "samwilson"
    ],
    "tools.erwin85": [
        "akoopal",
        "alphos",
        "cyberpower678",
        "hartman",
        "magnus",
        "mzmcbride",
        "nemobis",
        "pirsquared",
        "se4598"
    ],
    "tools.family": [
        "multichill",
        "odi",
        "multichill",
        "odi"
    ],
    "tools.fawikiauto": [
        "ebrahim",
        "reza",
        "ladsgroup"
    ],
    "tools.fiwiki-tools": [
        "zache-tool",
        "zache-tool"
    ],
    "tools.ftl": [
        "jmo",
        "jmo"
    ],
    "tools.geophotoreq": [
        "alexz"
    ],
    "tools.germancontributioncounts": [
        "aka"
    ],
    "tools.giftbot": [
        "gifti",
        "taxonbot",
        "gifti",
        "taxonbot",
        "gifti",
        "taxonbot"
    ],
    "tools.globalprefs": [
        "legoktm",
        "legoktm"
    ],
    "tools.gpy": [
        "gryllida",
        "gryllida"
    ],
    "tools.grillitus": [
        "hprmedina"
    ],
    "tools.hat-collector": [
        "deltaquad",
        "lixxx235",
        "maurelio",
        "mf-warburg",
        "rschen7754",
        "deltaquad",
        "lixxx235",
        "maurelio",
        "mf-warburg",
        "rschen7754"
    ],
    "tools.heimdall": [
        "dz"
    ],
    "tools.herculebot": [
        "hercule",
        "hercule"
    ],
    "tools.hewiki-tools": [
        "matanya",
        "eranroz"
    ],
    "tools.hoo": [
        "hoo"
    ],
    "tools.hroest": [
        "hroest"
    ],
    "tools.hrwiki": [
        "stemd"
    ],
    "tools.icalendar": [
        "wiegels"
    ],
    "tools.igloo": [
        "kangaroopower"
    ],
    "tools.imagemapedit": [
        "dapete",
        "dapete"
    ],
    "tools.inactiveadmins": [
        "aka",
        "aka"
    ],
    "tools.incolabot": [
        "incola"
    ],
    "tools.inkowik": [
        "inkowik",
        "inkowik"
    ],
    "tools.ipp": [
        "apper"
    ],
    "tools.itwp-deletions": [
        "mauro742",
        "mauro742"
    ],
    "tools.jackbot": [
        "jackpotte"
    ],
    "tools.jurgennl": [
        "jurgennl"
    ],
    "tools.jury": [
        "wiegels"
    ],
    "tools.krdbot": [
        "krd",
        "krd"
    ],
    "tools.lbenedix": [
        "lbenedix"
    ],
    "tools.legobot": [
        "legoktm"
    ],
    "tools.liangent-django": [
        "liangent"
    ],
    "tools.liangent-php": [
        "liangent"
    ],
    "tools.listpages": [
        "aka"
    ],
    "tools.lists": [
        "incola",
        "alessio",
        "fale",
        "incola",
        "alessio",
        "fale"
    ],
    "tools.locator": [
        "multichill",
        "akoopal",
        "silverspoon",
        "hartman"
    ],
    "tools.lonelylinks": [
        "sanyi4"
    ],
    "tools.lrbot": [
        "rluts"
    ],
    "tools.massaction": [
        "addshore"
    ],
    "tools.mathbot": [
        "mathbot"
    ],
    "tools.matvaretabellen": [
        "danmichaelo",
        "danmichaelo"
    ],
    "tools.mavrikant": [
        "mavrikant",
        "tools.mavrikant"
    ],
    "tools.merlbot2": [
        "merl",
        "tools.merlbot"
    ],
    "tools.mg-bot": [
        "miguillen"
    ],
    "tools.mrmetadata": [
        "ladsgroup",
        "gpaumier",
        "ladsgroup",
        "gpaumier"
    ],
    "tools.multichill": [
        "multichill"
    ],
    "tools.nlwikibots": [
        "siebrand",
        "multichill",
        "akoopal",
        "silverspoon",
        "valhallasw",
        "edoderoo",
        "siebrand",
        "multichill",
        "akoopal",
        "silverspoon",
        "valhallasw",
        "edoderoo"
    ],
    "tools.nn1l2bot": [
        "nn1l2"
    ],
    "tools.nullzerobot": [
        "nullzero"
    ],
    "tools.os": [
        "rhaworth"
    ],
    "tools.osm": [
        "kolossos",
        "dschwen",
        "retsam",
        "kolossos",
        "dschwen",
        "retsam"
    ],
    "tools.osm-add-tags": [
        "kolossos"
    ],
    "tools.para": [
        "para"
    ],
    "tools.pb": [
        "euku",
        "ireas",
        "wiegels",
        "tools.spbot"
    ],
    "tools.personabot": [
        "incola",
        "rotpunkt"
    ],
    "tools.phetools": [
        "tpt",
        "phe",
        "tpt",
        "phe",
        "tpt",
        "phe",
        "tpt",
        "phe"
    ],
    "tools.plantel2wiki": [
        "juanman",
        "juanman"
    ],
    "tools.pltools": [
        "pasleim",
        "pasleim"
    ],
    "tools.popularpages": [
        "kaldari",
        "alexz"
    ],
    "tools.potd-feed": [
        "daniel"
    ],
    "tools.powow": [
        "peter17"
    ],
    "tools.ptbot": [
        "arashpt"
    ],
    "tools.ptools": [
        "pleclown"
    ],
    "tools.ptwikis": [
        "alchimista",
        "danilo",
        "agx",
        "henriquecrang",
        "raylton-sousa",
        "rodrigopadula",
        "lestaty",
        "he7d3r",
        "diego"
    ],
    "tools.query2map": [
        "kolossos",
        "kolossos"
    ],
    "tools.railways": [
        "multichill"
    ],
    "tools.random-featured": [
        "dapete",
        "dapete"
    ],
    "tools.reftoolbar": [
        "kaldari",
        "alexz",
        "kaldari",
        "alexz"
    ],
    "tools.revertstat": [
        "aka",
        "aka"
    ],
    "tools.revisionstats": [
        "slaporte",
        "mahmoud"
    ],
    "tools.russbot": [
        "russblau",
        "russblau"
    ],
    "tools.sdbot": [
        "danmichaelo",
        "danmichaelo"
    ],
    "tools.searchsbl": [
        "seth"
    ],
    "tools.serviceawards": [
        "mormegil"
    ],
    "tools.shortnames": [
        "marc",
        "mdann52"
    ],
    "tools.simple": [
        "hydriz"
    ],
    "tools.simplewikt": [
        "hydriz"
    ],
    "tools.slumpartikel": [
        "ainali",
        "chm",
        "lokal-profil",
        "prolineserver"
    ],
    "tools.snaevar-bot": [
        "snaevar-bot"
    ],
    "tools.spellcheck": [
        "apper",
        "jem"
    ],
    "tools.srwiki": [
        "dungodung"
    ],
    "tools.stemmeberettigelse": [
        "danmichaelo"
    ],
    "tools.stimmberechtigung": [
        "ireas"
    ],
    "tools.styleguide": [
        "mattflaschen",
        "spage",
        "jdlrobson",
        "prtksxna"
    ],
    "tools.suggestbot": [
        "nettrom",
        "halfak",
        "kjschiroo",
        "stuem007"
    ],
    "tools.supercount": [
        "cyberpower678"
    ],
    "tools.svgtranslate": [
        "jarry1250"
    ],
    "tools.tb-dev": [
        "tb"
    ],
    "tools.tc-rc": [
        "tcn7jm",
        "revi"
    ],
    "tools.templatecount": [
        "jarry1250"
    ],
    "tools.test-lighttpd-precise": [
        "yuvipanda"
    ],
    "tools.theoslittlebot": [
        "theopolisme"
    ],
    "tools.tilde": [
        "mxn"
    ],
    "tools.timescale": [
        "smith609"
    ],
    "tools.tools-info": [
        "hedonil"
    ],
    "tools.toolschecker": [
        "marc",
        "yuvipanda",
        "andrew",
        "marc",
        "yuvipanda",
        "andrew"
    ],
    "tools.traffic-grapher": [
        "kevinpayravi",
        "kevinpayravi"
    ],
    "tools.translate": [
        "dereckson",
        "poulpy"
    ],
    "tools.tree-of-life": [
        "hoo",
        "lucie",
        "jzerebecki"
    ],
    "tools.tsreports": [
        "mzmcbride",
        "valhallasw",
        "tools.tsreports-dev",
        "mzmcbride",
        "valhallasw",
        "tools.tsreports-dev"
    ],
    "tools.unblockbot": [
        "beetstra"
    ],
    "tools.url-converter": [
        "seth"
    ],
    "tools.usergraph": [
        "pirsquared"
    ],
    "tools.usersearch": [
        "sigma",
        "scottywong",
        "sigma",
        "scottywong"
    ],
    "tools.usrd-tools": [
        "rschen7754",
        "happy5214",
        "tcn7jm",
        "fredddie",
        "rschen7754",
        "happy5214",
        "tcn7jm",
        "fredddie"
    ],
    "tools.veblenbot": [
        "cbm",
        "ruhrfisch",
        "tools.veblenbot"
    ],
    "tools.vtwo": [
        "v2",
        "v2"
    ],
    "tools.wahldiagramm": [
        "alexxw"
    ],
    "tools.weeklypedia": [
        "slaporte",
        "mahmoud"
    ],
    "tools.welcomebots-bn": [
        "tanvir",
        "taruuno",
        "tanvir",
        "taruuno",
        "tanvir",
        "taruuno",
        "tanvir",
        "taruuno"
    ],
    "tools.whymbot": [
        "whym"
    ],
    "tools.wikidata-janitor": [
        "multichill"
    ],
    "tools.wikidata-nolabels": [
        "ashig",
        "dereckson",
        "ashig",
        "dereckson"
    ],
    "tools.wikifactmine-api": [
        "tarrow"
    ],
    "tools.wikihistory": [
        "cyberpower678",
        "apper",
        "sigma",
        "elee",
        "cyberpower678",
        "apper",
        "sigma",
        "elee"
    ],
    "tools.wikipedia-contributor-locations": [
        "theopolisme"
    ],
    "tools.wikitrends": [
        "johang",
        "johang"
    ],
    "tools.wikitrip": [
        "sonet"
    ],
    "tools.wikivoyage": [
        "zhuyifei1999",
        "torty3",
        "atsirlin"
    ],
    "tools.wikiwatchdog": [
        "sonet"
    ],
    "tools.wikt-mwtest": [
        "darkdadaah",
        "jackpotte",
        "psychoslave"
    ],
    "tools.wiktioutils": [
        "darkdadaah"
    ],
    "tools.wiwosm": [
        "kolossos",
        "simon04",
        "retsam",
        "kolossos",
        "simon04",
        "retsam"
    ],
    "tools.wle": [
        "rluts"
    ],
    "tools.wlmtrafo": [
        "multichill",
        "spider"
    ],
    "tools.wm-bot": [
        "petrb",
        "petrb"
    ],
    "tools.wm-metrics": [
        "jeanfred",
        "lena",
        "pierreselim"
    ],
    "tools.wmukevents": [
        "jarry1250"
    ],
    "tools.xtools": [
        "cyberpower678",
        "matthewrbowker",
        "musikanimal",
        "technical-13",
        "cyberpower678",
        "matthewrbowker",
        "musikanimal",
        "technical-13",
        "cyberpower678",
        "matthewrbowker",
        "musikanimal",
        "technical-13",
        "cyberpower678",
        "matthewrbowker",
        "musikanimal",
        "technical-13"
    ],
    "tools.yadkard": [
        "dalba"
    ],
    "tools.yichengtry": [
        "vrandezo",
        "pasleim",
        "anorange0409",
        "gwiki"
    ],
    "tools.yifeibot": [
        "zhuyifei1999",
        "steinsplitter",
        "zhuyifei1999",
        "steinsplitter"
    ],
    "tools.ytcleaner": [
        "mbch331"
    ],
    "tools.zbot": [
        "alexz"
    ],
    "tools.zoomable-images": [
        "kolossos"
    ]
    "tools.zoomable-images": [
        "kolossos"
    ]
}
#!/usr/bin/python3

# Copyright 2016 Madhumitha Viswanathan mviswanathan@wikimedia.org
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
#
# Maps tool users to precise tools
# Run like tail -500000 /data/project/.system/accounting | ./precise-owners.py
# from tools bastions
#

import collections
import datetime
import fileinput
import http.client
import json
import ldap3
import operator
import yaml


def ldap_conn(config):
    """
    Return a ldap connection
    Return value can be used as a context manager
    """
    servers = ldap3.ServerPool([
        ldap3.Server(host)
        for host in config['servers']
    ], ldap3.POOLING_STRATEGY_ROUND_ROBIN, active=True, exhaust=True)
    return ldap3.Connection(servers, read_only=True,
        user=config['user'],
        auto_bind=True,
        password=config['password'])


def uid_from_dn(dn):
    keys = dn.split(',')
    uid_key = keys[0
    ]
    uid = uid_key.split('=')[1]
    return uid


def tools_members(config, tools):
    """
    Return a dict that has members of a tool associated with each tool
    Ex:
    {'tools.musikbot': ['musikanimal'],
     'tools.ifttt': ['slaporte', 'mahmoud', 'madhuvishy', 'ori']}
    """
    tool_to_members = collections.defaultdict(list)
    with ldap_conn(config) as conn:
        for tool in tools:
            conn.search(
                'ou=servicegroups,dc=wikimedia,dc=org',
                '(cn={})'.format(tool),
                ldap3.SEARCH_SCOPE_WHOLE_SUBTREE,
                attributes=['member', 'cn'],
                time_limit=5
            )
            for resp in conn.response:
                attributes = resp.get('attributes')
                members = attributes['member'] if attributes.get('member') else []
                tool_to_members[tool].extend([uid_from_dn(member) for member in members])
    return tool_to_members

def members_tools(tool_to_members):
    all_members = set().union(*tool_to_members.values())
    member_to_tools = collections.defaultdict(list)
    for tool, members in tool_to_members.items():
        for m in members:
            if tool not in member_to_tools[m]:
                member_to_tools[m].append(tool)
    return member_to_tools

def is_precise_host(hostname):
    if hostname[-4:].startswith('12'):
        return True

def grid_precise_tools():
    all_precise_tools = []
    conn = http.client.HTTPConnection('tools.wmflabs.org')
    conn.request("GET", "/gridengine-status",
                 headers={"User-Agent": "Precise tools finder|labs-admin@lists.wikimedia.org"})
    res = conn.getresponse().read().decode('utf-8')
    if res:
        grid_info = json.loads(res)["data"]["attributes"]
    for hostname, info in grid_info.items():
        if is_precise_host(hostname):
            if info["jobs"]:
                all_precise_tools.extend([job["job_owner"] for job in info["jobs"].values()])
    return all_precise_tools

def accounting_tools():
    DAYS=7
    FIELD_NAMES = [
        'qname', 'hostname', 'group', 'owner', 'job_name', 'job_number', 'account',
        'priority', 'submission_time', 'start_time', 'end_time', 'failed',
        'exit_status', 'ru_wallclock', 'ru_utime', 'ru_stime', 'ru_maxrss',
        'ru_ixrss', 'ru_ismrss', 'ru_idrss', 'ru_isrss', 'ru_minflt', 'ru_majflt',
        'ru_nswap', 'ru_inblock', 'ru_oublock', 'ru_msgsnd', 'ru_msgrcv',
        'ru_nsignals', 'ru_nvcsw', 'ru_nivcsw', 'project', 'department',
        'granted_pe', 'slots', 'task_number', 'cpu', 'mem', 'io', 'category',
        'iow', 'pe_taskid', 'maxvemem', 'arid', 'ar_submission_time',
    ]

    cutoff = (datetime.datetime.now() - datetime.timedelta(days=DAYS)).timestamp()
    precise_tools = []

    for line in fileinput.input():
        parts = line.split(':')
        job = dict(zip(FIELD_NAMES, parts))
        if int(job['end_time']) < cutoff:
            continue
        if 'release=precise' in job['category'] and job['owner'] not in precise_tools:
            precise_tools.append(job['owner'])

    return precise_tools

def member_emails(member_uids):
    uid_emails = collections.defaultdict(list)
    for uid in member_uids:
        with ldap_conn(config) as conn:
            conn.search('ou=people,dc=wikimedia,dc=org',
                        '(uid={})'.format(uid),
                        ldap3.SEARCH_SCOPE_WHOLE_SUBTREE,
                        attributes=['mail'],
                        time_limit=5
                        )
            for resp in conn.response:
                attributes = resp.get('attributes')
                uid_emails[uid] = attributes['mail'] if attributes.get('mail') else ''
    return uid_emails


def notify_admins(member_tools, member_emails):

    for member, tools in member_tools.items():
        email = member_emails[member]
        subject = '[weekly reminder][action required][Precise Tools need migration]'
        body = """
Ubuntu Precise was released in April 2012, and support for it
(including security updates) will cease in April 2017. We need to shut
down all Precise hosts before the end of support date to ensure that
Tool Labs remains a secure platform. This means all precise tools that are
currently launched with release=precise will no longer function starting
March 2017, and will crash with an error.

You are registered as admin/maintainer for the following tools, that are still
on precise: \n{}

Please make sure to migrate these over to Trusty as early as possible, to
ensure continued operation.

The steps to migrate to Trusty, and more information about the Precise deprecation
are here - https://wikitech.wikimedia.org/wiki/Tools_Precise_deprecation#What_should_I_do.3F.

Do feel free to reach out with questions/help at #wikimedia-labs.
    """.format('\n'.join(tools))

    args = ['/usr/bin/mail', '-s', subject, email]
    p = subprocess.Popen(args, stdout=subprocess.PIPE,
                         stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
    p.communicate(input=body)[0]

with open('/etc/ldap.yaml') as f:
    config = yaml.safe_load(f)

# tool_to_members = tools_members(config, accounting_tools() + grid_precise_tools())
tool_to_members = tools_members(config, ['tools.admin', 'tools.ifttt', 'tools.cbench'])
mt = members_tools(tool_to_members)
print(mt)
emails = member_emails(mt.keys())
# print json.dumps(mt, sort_keys=True, indent=4, separators=(',', ': '))
print(emails)
notify_admins(mt, emails)