diff --git a/database.py b/database.py index 64e48ae..2099749 100644 --- a/database.py +++ b/database.py @@ -1,212 +1,212 @@ # -*- coding: utf-8 -*- from twisted.internet import reactor from collections import deque, defaultdict from urllib2 import urlopen import time, json, re, oursql, os wikimediaCloaks = ('wikipedia/', 'wikimedia/', 'wikibooks/', 'wikinews/', 'wikiquote/', 'wiktionary/', 'wikisource/', 'wikivoyage/', 'wikiversity/', 'wikidata/', 'mediawiki/') def irclower(s): "IRC lower case: acording RFC 2812, the characters {}|^ are the respective lower case of []\~" return s.lower().replace('{', '[').replace('}', ']').replace('|', '\\').replace('^', '~') def irccompare(a, b): "Compare two nicks using IRC rules to lower case, e.g. '[abc]' == '{abc}'" return a == b or irclower(a) == irclower(b) mask2re = {'\\': '[\\\\|]', '|': '[\\\\|]', '^': '[~^]', '~': '[~^]', '[': '[[{]', ']': '[]}]', '{': '[[{]', '}': '[]}]', '*': '[^!@]*', '?': '[^!@]', '+': '\\+', '.': '\\.', '(': '\\(', ')': '\\)', '$': '\\$'} def maskre(mask): "Transforms an IRC mask into a regex pattern" mask = irclower(mask) r = '' for c in mask: r += mask2re.get(c, c) try: return re.compile(r + '$', re.I) except: return None def mklist(items): "Tranforms a sequence ['a', 'b', 'c'] into 'a, b and c' " if type(items) != list: items = list(items) return len(items) > 1 and ', '.join(items[:-1]) + ' and ' + items[-1] or items and items[0] or '' def addhook(hook, func, *errhooks): "Add a hook to hooks list" hookList.append((time.time(), hook, func) + errhooks) def callhook(hook): "Calls all correspondent hooks" with open('hooks.log', 'a') as f: f.write('[%s] %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), hook)) for item in list(hookList): t, _hook, func = item[:3] if t + 10 < time.time(): hookList.remove(item) elif hook == _hook: func() hookList.remove(item) elif len(item) > 3 and hook in item[3:]: func(hook) hookList.remove(item) class dbConn: "Connect to bot database" def __init__(self): self.connection = None self.connect() def connect(self): try: self.connection.close() except: pass self.connection = oursql.connect(db='s53213__wmopbot', host='tools.labsdb', - read_default_file=os.path.expanduser('~/replica.my.cnf'), read_timeout=15, use_unicode=False, + read_default_file=os.path.expanduser('~/replica.my.cnf'), read_timeout=20, use_unicode=False, autoreconnect=True, autoping=True) self.cursor = self.connection.cursor() self.last = time.time() def execute(self, sql, params=()): try: self.cursor.execute(sql, params) self.last = time.time() except oursql.OperationalError: # probably connection lost if self.last + 10 < time.time(): self.connect() self.cursor.execute(sql, params) def query(self, sql, params=()): self.execute(sql, params) return self.cursor.fetchall() def _initLists(): "Populate data lists with SQL database data" global userFlags sep = re.compile(r'^;|(?<=[^\\]);') db = dbConn() # list: channel (50), type (10), target (90), args (300), timestatmp (datetime) for channel, _type, target, args in db.query('SELECT * FROM list'): chanList[channel].setdefault(_type, {})[target] = \ {k: v.replace('\;', ';') for k, v in (i.split(':', 1) for i in sep.split(args))} userFlags = {u: set(chanList['global']['flags'][u]['flags'].split()) for u in chanList['global'].get('flags', ())} # lang: key(30), message(255), user(16), timestamp for lg_key, msg in db.query('SELECT lg_key, lg_message FROM lang'): lang, key = lg_key.split('.', 1) msgLang.setdefault(key, {})[lang] = msg # $j bans for chan in chanList: for target in chanList[chan]['ban']: if target.startswith('$j:#'): jBans[irclower(target[3:])].append(chan) def dbSync(force=False): "Save data in database" start = time.time() now = time.strftime('%Y-%m-%d %H:%M:%S') # list: channel (50), type (10), target (90), args (300) current = {(chan, _type, target, ';'.join(':'.join((k, v.replace(';', '\;'))) for k, v in chanList[chan][_type][target].items())) for chan in chanList for _type in chanList[chan] for target in chanList[chan][_type]} indb = set(db.query('SELECT * FROM list')) delete = indb - current insert = list(current - indb) del current, indb # free memory with open('dbupdates.log', 'a') as f: if delete: f.write('[%s] DELETE FROM list\n' % time.strftime('%Y-%m-%d %H:%M:%S')) f.writelines(repr(line) + '\n' for line in delete) db.execute('DELETE FROM list WHERE (ls_channel, ls_type, ls_target) IN (%s)' % ','.join( '(?,?,?)' for d in delete), tuple(i for d in delete for i in d[:3])) if insert: f.write('[%s] INSERT INTO list\n' % time.strftime('%Y-%m-%d %H:%M:%S')) f.writelines(repr(line) + '\n' for line in insert) n = 0 while n < len(insert): db.execute('INSERT INTO list VALUES %s' % ','.join('(?,?,?,?)' for n in insert[n:n + 1000]), tuple(i for x in insert[n:n + 1000] for i in x)) n += 1000 uploadusers = [u.toDB() for u in nickUser.itervalues() if not u.issaved] while uploadusers: part = uploadusers[:50] del uploadusers[:50] db.execute('REPLACE INTO user VALUES ' + ','.join(('(?,?,?,?,NOW(),?,?)',) * len(part)), tuple(i for u in part for i in u)) def dumpArgs(d): return ';'.join(':'.join((k, v.replace(';', '\;'))) for k, v in d.items()) def regAction(user, channel, action, target, args=None): "insert action into db" # actions: id(int), user(90), chan(50), action(10), target(90), args(255), timestamp(datetime) db.query('INSERT INTO actions (ac_user, ac_chan, ac_action, ac_target, ac_args) (?, ?, ?, ?, ?)', (user, channel, action, target, args)) def langMsg(lang, msg, *args): if msg in msgLang: if isinstance(lang, dict): lang = lang.get('lang', 'en') elif lang.startswith('#'): lang = chanList[lang]['config'].get('LANG', {}).get('prefix', 'en') message = lang in msgLang[msg] and msgLang[msg][lang] or msgLang[msg].get('en') if not message: return msg if args and message.count('%s') == len(args): return message % args return message return msg def getWmChannels(): global wmChannels api = urlopen('https://meta.wikimedia.org/w/api.php?action=query&format=json&prop=revisions&titles=IRC/Channels&rvprop=content') data = json.loads(api.read()) text = data['query']['pages']['6237']['revisions'][0]['*'] wmChannels = {'#' + str(chan).lower() for chan in re.findall(r'\n\| ?\{\{[Cc]hannel\|([^ |}]+)\}\}', text)} for chan in wmChannels: if not chan in chanList: chanList[chan]['config']['WM'] = {'time': '%d' % time.time(), 'user': 'wmopbot', 'comment': 'got automaticaly from https://meta.wikimedia.org/wiki/IRC/Channels'} for chan in chanList: if chan not in wmChannels: del chanList[chan]['config']['WM'] def statsMsg(channel): "increment channel message statistics" hour = int(time.time()) / 3600 hourChanStats[hour][channel] += 1 def ststsUser(channel, num): "register number of users in channel" now = int(time.time()) last = chanUserStats[channel] and chanUserStats[channel][-1] if last and last[0] + 10 > now: chanUserStats[channel][-1] = (now, num) else: chanUserStats[channel].append((now, num)) if not 'chanNicks' in globals(): db = dbConn() chanNicks = defaultdict(dict) nickUser = {} chanConfig = defaultdict(dict) chanList = defaultdict(lambda: {'access': {}, 'ban': {}, 'quiet': {}, 'mode': {}, 'exempt': {}, 'config': {}}) msgLang = defaultdict(dict) offUsers = [] blackList = [] jBans = defaultdict(list) commQueue = defaultdict(list) opCommands = deque(maxlen=10) recentSpam = deque(maxlen=10) hourChanStats = defaultdict(lambda: defaultdict(int)) hookList = [] topics = {} getWmChannels() _initLists() else: print '[%s] Reloaded database.py' % time.strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools.py b/tools.py index 0698230..61f5116 100644 --- a/tools.py +++ b/tools.py @@ -1,2385 +1,2385 @@ #!/use/bin/python # -*- coding: utf-8 -*- """ wmOpBot tools. @author: danilo (freenode), [[Usuário:Danilo.mac]] (wiki) @licence: GNU General Public License 3.0 (GPL V3) """ import time, re, sys, requests, thread, json from collections import deque from urllib2 import urlopen from database import * from requests_oauthlib import OAuth1 from user import User from urllib2 import urlopen import warns allowJoin = True calledops = set() sigynDefcon = None if not '_whois' in globals(): _whois = {} whoisQueue = [] lastWhois = 0 nsQueue = [] lastNsInfo = 0 setMode = {} def prnt(x): "for debug" print x if not 'waitingSupport' in globals(): waitingSupport = {} def isop(user, channel): account = user.account if isinstance(user, dict) else user if 'o' in chanList[channel]['access'].get(account, {}).get('flags', '').lower(): return True chanacs = [i for i in chanList[channel]['access'] if i.startswith('$chanacs:')] for entry in chanacs: chan = entry[9:] if chan in chanList and 'o' in chanList[channel]['access'][entry]['flags'].lower() \ and chanList[chan]['access'].get(account, {}).get('flags') not in (None, 'b'): return True return False class Command: "Class to process bot commands" def __init__(self, command, user, channel): self.word = command.split() if self.word[0][-2:] == '\'s': self.word[0:1] = [self.word[0][0:-2], 'is'] cmdword = self.word[0].upper().strip('!') if channel == 'pm' and hasattr(self, 'pm_' + cmdword): cmd = getattr(self, 'pm_' + cmdword) elif hasattr(self, 'wmop_' + cmdword) and isop(user, '#wikimedia-ops') and channel in \ ('#wikimedia-ops', '#wikimedia-ops-internal', '#wikimedia-opbot', 'pm'): cmd = getattr(self, 'wmop_' + cmdword) elif channel == 'pm': self.isvalid = False return elif hasattr(self, 'chanop_' + cmdword) and isop(user, channel): cmd = getattr(self, 'chanop_' + cmdword) elif hasattr(self, 'cmd_' + cmdword): cmd = getattr(self, 'cmd_' + cmdword) else: self.isvalid = False print 'Not a valid command by %s: %s' % (user.full, command) return self.ispm = channel == 'pm' self.response = self.ispm and user.nick or channel self.channel = channel self.user = user self.isvalid = user['host'].startswith(wikimediaCloaks) or 'account' in user and isop(user, '#wikimedia-ops') \ or self.ispm and (command == 'cloak' or command.startswith('botcloak ')) if not self.isvalid: return self._run = cmd self.runCount = 0 self.action = {} def run(self): "run funtion preventing loop" self.runCount += 1 if self.runCount > 2: print '[%s] Error: loop detected in Command.run (command: %s)' % (time.strftime('%Y-%m-%d %H:%M:%S'), ' '.join(self.word)) return if not 'account' in self.user and self.runCount == 1: addhook('whois ' + self.user.nick, self.run) bot.sendLine('WHOIS ' + self.user.nick) else: self._run() def msg(self, m, *args, **kw): "Expand the messages codes according the respective channel language" message = langMsg(not self.ispm and self.response in chanList and self.response or 'en', m, *args) if kw.get('notice'): bot.notice(kw.get('prefix', '') + self.response, message) else: bot.msg(kw.get('prefix', '') + self.response, message) def cmd_OPS(self): "Call ops" channel = len(self.word) > 1 and self.word[1][0] == '#' and self.word[1] or self.channel if self.ispm and not channel in chanList or channel in ('#wikimedia-ops', '#wikimedia-ops-internal'): return if channel == self.channel == '#wikipedia-en-helpers': channel = '#wikipedia-en-help' logAction(user=self.user, channel='#wikimedia-ops', action='!ops', target=channel) if not self.ispm and 'AntiSpamMeta' in chanNicks[channel] or channel in calledops: return comment = len(self.word) > 1 + (self.channel != channel) and ' (%s)' % ' '.join( self.word[1 + (self.channel != channel):]) or '' ping = warns.mkping(channel, noflag='n') callChan = channel.startswith('#wikimedia-opbot') and '#wikimedia-opbot' or '#wikimedia-opbot,#wikimedia-ops' bot.msg(callChan, '%s wants ops attention in \x02%s\x02%s%s' % (self.user.nick, channel, comment, ping and ' \x0315ping ' + mklist(sorted(ping)) or '')) calledops.add(channel) def pm_OPS(self): "Make cmd_OPS also work in pm" self.cmd_OPS() def cmd_GCS(self): "Call GCs" if self.ispm or not self.response in ('#wikimedia-ops', '#wikimedia-ops-internal', '#wikimedia-opbot'): return accounts = [a for a in chanList['#wikimedia-ops']['access'] if chanList['#wikimedia-ops']['access'][a].get('template') == 'Contact'] gcs = [n for n in chanNicks[self.channel] if nickUser[n].account in accounts] self.msg('ping ' + mklist(gcs)) def kbqInit(self): "initial checks for KB and QUIET commands" print 'kbqInit' if len(self.word) < 2: self.msg('wrongsyntax') return nick = self.word[1] if not isop(self.user, self.channel): self.msg('notop') return False if not isop('wmopbot', self.channel) and \ not [1 for nick in bot.nicks if (nick, 2) in chanNicks[self.channel].items()]: self.msg('iamnotop') return False if not self.channel in chanNicks: self.msg('iamnotinchan') return if not nick in chanNicks[self.channel]: self.msg('nicknotinchan') return params = ' '.join(self.word[2:]).rsplit('~', 1) self.action.update({'comment': params[0], 'expiry': len(params) == 2 and params[1] or '', 'user': self.user, 'channel': self.channel, 'note': ' '.join(self.word)}) self.action['target'] = target = nickUser[nick] if isop(target, '#wikimedia-ops'): self.msg('notforwmops') return if target.host.startswith('freenode/staff'): self.msg('notforstaff') return if target.host.startswith(wikimediaCloaks): self.msg('notforwmcloak') return print 'end kbqInit' if not 'ip' in target and not 'account' in target: addhook('whois ' + nick, self.run) bot.sendLine('WHOIS ' + nick) else: return True def cmd_KB(self): "Command !kb nick | KB #channel nick" print 'cmd_KB' if self.ispm: return if len(self.word) > 2 and '#' in self.word[2][0] + self.word[1][0]: self.kb_chan() return if not self.kbqInit(): return logAction(self.action, action='!kb') self.kickban() def kb_chan(self): "Kickban requested from #wikimedia-ops" print 'kb_chan' if not self.channel in ('#wikimedia-ops', '#wikimedia-ops-internal', '#wikimedia-opbot'): print 'kb_chan in ' + self.channel return if not isop(self.user, '#wikimedia-ops'): self.msg('onlywmops') return if self.word[1][0] == '#': nick, channel = self.word[2], self.word[1] else: nick, channel = self.word[1], self.word[2] if not channel in chanNicks: self.msg('iamnotinchan') return if not nick in chanNicks[channel]: self.msg('nicknotinchan') return if not isop('wmopbot', channel) and not [n for n in bot.nicks if chanNicks[channel].get(n) == 2]: self.msg('I am not op in that channel') return ischanop = isop(self.user, channel) if not ischanop and self.channel in waitingSupport: self.msg('There is already a propose waiting !support/!oppose (%s), other proposes can be done only afer this is closed or expirated' % waitingSupport[self.channel]['propose']) return params = ' '.join(self.word[3:]).rsplit('~', 1) print params self.action.update({'comment': params[0].strip(), 'channel': channel}) self.action['user'] = self.user self.action['expiry'] = exp = len(params) == 2 and params[1].strip() or '1h' self.action['target'] = target = nickUser[nick] if not 'ip' in target and not 'account' in target: addhook('whois ' + nick, self.run) bot.sendLine('WHOIS ' + nick) if isop(target, '#wikimedia-ops'): self.msg('notforwmops') return if target.host.startswith('freenode/staff'): self.msg('notforstaff') return try: expiry = exp.strip() == 'p' and ' permanently' or ' for %s %s%s' % (exp[:-1].strip(), {'m': 'minute', 'h': 'hour', 'd': 'day'}[exp[-1]], int(exp[:-1]) != 1 and 's' or '') except: self.msg('wrongexpiry') return if ischanop: logAction(self.action, channel=self.channel, action='!kb #chan', target=' '.join(self.word[1:3])) self.kickban() if channel in waitingSupport and waitingSupport[channel]['target'] == target: del waitingSupport[channel] return propose = 'kickban %s from %s' % (target.nick, channel) def applyBan(): self.action['note'] = 'requested by ' + \ mklist([u.nick for u in waitingSupport[self.channel]['suporters']]) self.action['user'] = mklist(['%(nick)s (%(account)s)' % u for u in waitingSupport[self.channel]['suporters']]) self.kickban() waitingSupport[self.channel] = {'propose': propose, 'time': time.time(), 'suporters': [self.user], 'target': target, 'channel': channel, 'apply': applyBan, 'command': self} self.msg('%s is proposing to %s, I need 2 !support to apply or 1 !oppose to cancel' % (self.user.nick, propose + (self.action['comment'] and ' (%s)' % self.action['comment'] or '') + expiry)) logAction(self.action, channel=self.channel, action='!kb (prop)', target=' '.join(self.word[1:3])) def kickban(self): "Process kickban after first checks and whois query processed" print 'kickban', self.action channel = self.action['channel'] target = self.action['target'] self.action['ban'] = ban = 'account' in target and '$a:' + target.account or 'ip' in target and \ (('kiwiirc' in target.host or 'gateway/web/freenode' in target.host) and '*!*@*' or '*!*@') + \ target['ip'] or '*!*@' + target.host self.action['action'] = 'kickban' if not checkban(ban): bot.msg('internalerror') return def oped(): bot.sendLine('MODE %s +b %s' % (channel, ban)) bot.sendLine('KICK %s %s :%s' % (channel, target.nick, self.action.get('reason', ''))) opCommands.append(self.action) if 'eir' in chanNicks[channel]: try: eirComment[('ban', ban)] = (self.action.get('comment') or 'requested by ' + (isinstance(self.action['user'], dict) and self.action['user'].full or self.action['user']), self.action.get('expiry')) except: print 'Error on set eir comment %r' % self.action print 'end kickban' if [1 for nick in bot.nicks if (nick, 2) in chanNicks[channel].items()]: oped() else: addhook('op ' + channel, oped) bot.sendLine('CS OP ' + channel) reactor.callLater(3, bot.sendLine, 'CS DEOP ' + channel) def cmd_QUIET(self): "Command !quiet " print 'cmd_QUIET' if self.ispm: return if not self.kbqInit(): return logAction(self.action, action='!quiet') self.quiet() def quiet(self): "Process quiet after first checks and whois query processed" channel = self.action['channel'] target = self.action['target'] self.action['quiet'] = quiet = 'account' in target and '$a:' + target.account or 'ip' in target and \ (('kiwiirc' in target.host or 'gateway/web/freenode' in target.host) and '*!*@*' or '*!*@') + \ target['ip'] or '*!*@' + target.host if not checkban(quiet): bot.msg('internalerror') return opCommands.append(self.action) if 'eir' in chanNicks[channel]: try: eirComment[('quiet', quiet)] = (self.action.get('comment') or 'requested by ' + (isinstance(self.action['user'], dict) and self.action['user'].full or self.action['user']), self.action.get('expiry')) except: print 'Error on set eir comment %r' % self.action botnick = sorted([n for n in bot.nicks if n in chanNicks[self.channel]])[0] def oped(): bot.sendLine('MODE %s +q-o %s %s' % (self.channel, quiet, botnick)) if chanNicks[self.channel][botnick] == 2: bot.sendLine('MODE %s +q %s' % (self.channel, quiet)) else: addhook('op ' + self.channel, oped) bot.sendLine('CS OP ' + self.channel) def wmop_SUPPORT(self): "Command to support a proposal" if not self.channel in waitingSupport: self.msg('There is not a proposal waiting support') return prop = waitingSupport[self.channel] if not prop['target'].nick in chanNicks[prop['channel']]: self.msg('%s is not in %s, the proposal is cancelled' % (prop['target'].nick, prop['targetchan'])) del waitingSupport[self.channel] return if not isop(self.user, '#wikimedia-ops'): self.msg('onlywmops') return if self.user.account in [u.account for u in prop['suporters']]: self.msg('You are already a supporter, I need more %d to apply' % (3 - len(prop['suporters']))) return prop['suporters'].append(self.user) if len(prop['suporters']) > 2: self.msg('Applying') prop['apply']() del waitingSupport[self.channel] else: self.msg('Now I need more %d !support to apply' % (3 - len(prop['suporters']))) logAction(user=self.user, channel=self.channel, action='!support', target=prop['propose']) def wmop_OPPOSE(self): "Command to opose to a proposal" if not self.channel in waitingSupport: self.msg('There is not a proposal waiting suport') return if not isop(self.user, '#wikimedia-ops'): self.msg('onlywmops') return logAction(user=self.user, channel=self.channel, action='!oppose', note=waitingSupport[self.channel]['propose']) del waitingSupport[self.channel] self.msg('Proposal canceled') def cmd_MODE(self): "Set channel modes" if self.ispm or not self.channel.startswith('#wikimedia-opbot') or len(self.word) < 2: return channel = self.channel params = self.word[1:] delta = None for i, p in enumerate(params): if p[0] == '~': delta = ' '.join(params[i:]) modes = ' '.join(params[0:i]) break else: modes = ' '.join(params) if delta: m = re.search(r'~ ?(\d+)([smh])', delta) delta = m and int(m.group(1)) * {'s': 1, 'm': 60, 'h': 3600}[m.group(2)] if (channel, modes) in setMode and setMode[channel, modes].active(): setMode[channel, modes].cancel() def applymode(): if chanNicks[channel]['wmopbot'] >= 2: bot.sendLine('MODE %s %s' % (channel, modes)) else: def opped(): bot.sendLine('MODE %s %s' % (channel, modes)) bot.sendLine('MODE %s -o wmopbot' % channel) addhook('op ' + channel, opped) bot.sendLine('CS OP ' + channel) setMode[channel, modes] = reactor.callLater(delta or 1, applymode) if delta: self.msg('Mode change scheduled') def wmop_WHOWAS(self): "Command !info ..." if len(self.word) == 1: return target = None kicked = None # !whowas $a:account if self.word[1][0:3] == '$a:': r = db.query('SELECT * FROM user WHERE us_account = ? LIMIT 100', (self.word[1][3:],)) if not r: self.msg('I never saw an user with that account') return # !whowas X!Y@Z elif re.match(r'[^!@].*[!@].*[^!@]$', self.word[1]) and ('*' in self.word[1] or '?' in self.word[1]): mask = self.word[1].replace('%', r'\%').replace('_', r'\_').replace('*', '%').replace('?', '_') r = db.query('SELECT * FROM user WHERE us_user LIKE ? LIMIT 100', (mask,)) if not r: self.msg('I never saw an user that fit that mask') return kicked = db.query("SELECT COUNT(*) FROM actions WHERE ac_action = 'kick' AND ac_target LIKE ?", (mask,)) # !whowas 1.2.3.4; !info *!*@1.2.3.4 elif '.' in self.word[1] or ':' in self.word[1]: match = re.match(r'(?i)\d{1,3}(?:\.[*\d]{1,3}){3}|[0-9a-f]{1,4}(?:::?[0-9a-f*]{1,4}){1,8}(?:::)?', self.word[1]) if match: if '*' in match.group(0): r = db.query('SELECT * FROM user WHERE us_ip LIKE ? LIMIT 100', (match.group(0).replace('*', '%'),)) else: r = db.query('SELECT * FROM user WHERE us_ip = ? LIMIT 100', (match.group(0),)) if not r: self.msg('I never saw an user with that IP') return else: self.msg('Invalid IP') return # !whowas elif re.match(r'(?i)[][a-z0-9\\\|{}^~]{1,16}', self.word[1]): mask = self.word[1] + '!%' r = db.query('SELECT * FROM user WHERE us_user LIKE ? LIMIT 100', (mask,)) user = findUser(self.word[1]) if user: r.extend(db.query('SELECT * FROM user WHERE us_user LIKE ? LIMIT 100', ('%%!%(ident)s@%(host)s' % user,))) if not r: self.msg('I never saw an user with that nick' + (user and ' or same ident@host' or '')) return masks = (mask, '%%!%(ident)s@%(host)s' % user) if user else (mask,) where = '(ac_target LIKE ? OR ac_target LIKE ?)' if user else 'ac_target LIKE ?' kicked = db.query("SELECT COUNT(*) FROM actions WHERE ac_action = 'kick' AND " + where, masks) else: self.msg('wrong syntax') return nicks = {u[0][:u[0].find('!')] for u in r} idents = {u[0][u[0].find('!') + 1:u[0].find('@')] for u in r} hosts = {u[0][u[0].find('@') + 1:] for u in r} accounts = {u[1] for u in r if u[1]} seen = sorted(u[4] for u in r)[-1].isoformat(' ') score = sum(u[5] or 0 for u in r) stime = score > 144 and (score / 144, 'day') or (score / 6, 'hour') msg = 'nicks: %s; idents: %s; hosts: %%s; accounts: %s;' % \ tuple(mklist(len(l) > 5 and list(l)[:4] + ['%d others' % (len(l) - 4)] or l or ['(none)']) for l in (nicks, idents, accounts)) msg = msg % mklist(len(hosts) > 3 and list(hosts)[:2] + ['%d others' % (len(hosts) - 2)] or hosts or ['(none)']) if kicked and kicked[0][0]: msg += ' kicked %d time%s;' % (kicked[0][0], kicked[0][0] != 1 and 's' or '') msg += ' last seen: %s; summed seen time: %d %s%s' % (seen, stime[0], stime[1], stime[0] != 1 and 's' or '') self.msg(msg) def info_chan(self, channel): "Info about channels" nicks = chanNicks.get(channel, {}).viewkeys() - {'ChanServ'} lists = channel in chanList and any(chanList[channel].itervalues()) and chanList[channel] if not nicks and not lists: self.msg('idkthischan') return resp = nicks and '%d users now in channel, ' % len(nicks) or 'I am not in that channel, ' ops = [u for u in lists['access'] if 'o' in lists['access'][u].get('flags', '').lower()] resp += 'cserror' in lists['config'] and 'ChanServ query error: ' + lists['config']['cserror'] or \ '%d operator%s in access list (%s)%s' % (len(ops), len(ops) != 1 and 's' or '', 'wmopbot' in ops and 'I\'m op' or 'I\'m not op', nicks and ', %d now in channel' % sum(n in nickUser and nickUser[n].account in ops for n in nicks) or '') self.msg(resp) def wmop_WHERE(self): "Command to show the channels some nick are" user = None if len(self.word) == 2: target = self.word[1] elif len(self.word) == 3 and 'is' in self.word[1:3]: self.word.remove('is') target = self.word[1] elif len(self.word) == 4 and 'is' in self.word[1:3] and self.word[3].strip('?') == 'banned': self.word.remove('is') self.word = ['checkban', self.word[1]] return self.wmop_CHECKBAN() elif len(self.word) == 4 and 'is' in self.word[1:3] and self.word[3].strip('?') in ('opped', 'voiced'): status = self.word[3].strip('?') == 'opped' and 1 or 0 self.word.remove('is') user = findUser(self.word[1]) nick = user and user.nick or self.word[1] chans = {c for c in chanNicks if chanNicks[c].get(nick, 0) > status} self.msg('in ' + mklist(len(chans) > 12 and chans[0:10] + ['%d other channels' % (len(chans) - 10)] or chans) if chans else 'in no channel that I am in') return elif len(self.word) == 4 and 'is' in self.word[1:3] and self.word[3].strip('?') == 'op': self.word.remove('is') chans = wherehasflag(findUser(self.word[1]) or self.word[1], 'o') n = len(chans) msg = n and mklist(chans) while n and len(msg) > 375: n -= 1 msg = mklist(chans[0:n] + ['%d other channels' % (len(chans) - n)]) self.msg(msg or 'in no channel that I know') return elif len(self.word) == 4 and 'has' in self.word[1:3] and self.word[3].startswith('+'): flag = self.word[3][1:] chans = wherehasflag(findUser(self.word[1]) or self.word[1], flag) n = len(chans) msg = n and mklist(chans) while n and len(msg) > 375: n -= 1 msg = mklist(chans[0:n] + ['%d other channels' % (len(chans) - n)]) self.msg(msg or 'in no channel that I know') return else: self.msg('wrongsyntax') return if re.match(r'[^!@]+![^!@]+@[^!@]+', target): mask = maskre(target) if not mask: self.msg('Not a valid mask') return nicks = [u.nick for u in nickUser.itervalues() if mask.match(u.full)] if len(nicks) > 1: inChan = {n for c in chanNicks for n in chanNicks[c]} & set(nicks) if not inChan: self.msg('I don\'t see users that fit that mask') return if len(inChan) > 1: self.msg('There are more than one user that fit that mask: %s' % mklist( len(inChan) > 6 and list(inChan)[:5] + ['%d more' % len(inChan) - 5] or inChan)) return target = inChan.pop() elif len(nicks) == 1: user = nickUser[nicks.pop()] else: self.msg('I don\'t see users that fit that mask') return if not user: user = findUser(target) if user: if isop(user, '#wikimedia-ops'): self.msg('notforwmops') return if user.host.startswith('freenode/staff'): self.msg('notforstaff') return - if user.host.startswith(wikimediaCloaks): - self.msg('notforwmcloak') + if '/' in user.host and not user.host.startswith(('gateway', 'unaffiliated')) and not '/bot/' in user.host: + self.msg('That command can not be applied to affiliated users') return target = user.nick chans = [c for n, c in sorted([(len(chanNicks[chan]), chan) for chan in chanNicks if target in chanNicks[chan]])] if not chans: self.msg('I don\'t see %s in any channel' % target) else: n = len(chans) msg = n and '%s is in %s' % (target, mklist(chans)) while n and len(msg) > 375: n -= 1 msg = '%s is in %s' % (target, mklist(chans[0:n] + ['%d other channels' % (len(chans) - n)])) self.msg(msg or 'in no channel that I know') def wmop_BLOCKS(self): "Command to list users witk wikiblocks and the user wikiblocks" if not isop(self.user, '#wikimedia-ops'): self.msg('onlywmops') return if len(self.word) == 1: self.msg('wrongsyntax') return start = time.time() if self.word[1].startswith('hex:'): try: ip = reIP.match('.'.join(str(int(self.word[1][x:x + 2], 16)) for x in (4, 6, 8, 10))) except: self.msg('Invalid hexadecimal IP') return else: ip = reIP.search(self.word[1]) if ip: ip = ip.group(0).replace('-', '.') blocks = wikiBlock(ip) if blocks: self.msg('%s is blocked in %s' % (ip, blocks)) else: self.msg('No wiki blocks found for ' + ip) elif self.word[1][0] == '#': chan = self.word[1] if not chan in chanNicks: self.msg('iamnotinchan') return wikis = len(self.word) > 2 and self.word[2:] or None if wikis: nicks = [nick for nick in chanNicks[chan] if nick in nickUser and 'ip' in nickUser[nick] and any(w in (wikiBlock(nickUser[nick]['ip']) or ()) for w in wikis)] else: nicks = [nick for nick in chanNicks[chan] if nick in nickUser and 'ip' in nickUser[nick] and wikiBlock(nickUser[nick]['ip'])] if nicks: self.msg('Users in %s with wikiblocks: %s' % (chan, mklist(sorted(nicks)))) else: self.msg('No user with wikiblock in ' + chan) else: user = findUser(self.word[1]) if not user: self.msg('idkthisnick') return if not 'ip' in user: self.msg('I don\'t know this user\'s IP') return blocks = wikiBlock(user['ip']) if blocks: self.msg('%s IP %s is blocked in %s' % (user.nick, user['ip'], blocks)) else: self.msg('%s has no wiki block' % user.nick) print self.word, time.time() - start def wmop_CHECKBAN(self): "generate a list of bans that cover a user!ident@host or of users in a channel with bans in some other channel" if len(self.word) == 1: self.msg('wrongsyntax') return start = time.time() if self.word[1][0] == '#': channel = self.word[1] if not channel in chanNicks: self.msg('iamnotinchan') return match = {} for nick in chanNicks: if nick not in nickUser: continue user = nickUser[nick] for chan in chanBans: for ban, mask in chanBans[chan]: if mask.match(user.full): match.setdefault(nick, []).append(chan) if match: bans = sorted(['%s (%s)' % (nick, ', '.join(match[nick])) for nick in match], key=lambda i:not '#wikimedia-bans' in i) if len(bans) > 5: bans = bans[:4] + ['and %d other users' % (len(bans) - 4)] self.msg('Users in %s with bans in other channels: %s' % (channel, ', '.join(bans))) else: self.msg('No users in %s are banned in other channels' % channel) else: user = findUser(self.word[1]) if not user and self.word[1].startswith('hex:'): try: target = '.'.join(str(int(self.word[1][x:x + 2], 16)) for x in (4, 6, 8, 10)) except: self.msg('Invalid hexadecimal IP') return else: target = user and user.full or self.word[1] if not user and re.match(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', target): target = '*!*@' + target account = user and user.account ipban = re.compile(r'(?<=^\*!\*@)\d{1,3}\.\d{1,3}\.[*\d]{1,3}\.[*\d]{1,3}(?:/\d\d)?$') ip = ipban.search(target) or reIP.search(target) ip = ip and ip2numcdir(ip.group(0)) match = {} for chan in chanList: for ban in chanList[chan]['ban']: if ban[0:3] == '$a:' and ban[3:] == account: match.setdefault(ban, []).append(chan) continue if ipban.search(ban): if ip and testip(ip, ban[4:]): match.setdefault(ban, []).append(chan) continue mask = maskre(ban) if mask and mask.match(target): match.setdefault(ban, []).append(chan) if match: if len(match) == 1 and len(match.values()[0]) == 1: ban = match.keys()[0] chan = match[ban][0] args = chanList[chan]['ban'][ban] date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(args['time']))) \ if args.get('time', '').isdigit() else '(unknown date)' comm = 'comment' in args and ' with comment "%s"' % args['comment'] or '' self.msg('%s is banned in %s (%s set by %s on %s%s)' % (target, chan, ban, args['user'], date, comm)) return bans = sorted(['%s (%s)' % (', '.join(match[ban]), ban) for ban in match], key=lambda i:not '#wikimedia-bans' in i) if len(bans) > 5: bans = bans[:4] + ['and %d other bans' % (len(bans) - 4)] self.msg('%s is banned in %s' % (target, ', '.join(bans))) else: self.msg('No bans found for ' + target) print self.word, time.time() - start def wmop_REDUNDANT(self): "check for redundant bans in a channel" if len(self.word) == 1: self.msg('wrongsyntax') return if self.word[1][0] == '#': channel = self.word[1] if not channel in chanList: self.msg('idkthischan') return channel = self.word[1] check = [(ban, maskre(ban), chan) for chan in [channel] + [c for c in jBans if channel in jBans[c] and c in chanBans] for ban in chanList[chan]['ban'] if not ban.startswith('$j')] redundant = ['%s -> %s' % (ban2check, ban + (checkChan != channel and ' in ' + checkChan or '')) for ban, mask, checkChan in check for ban2check in chanList[channel]['ban'] if (checkChan != channel if ban == ban2check else mask.match(ban2check))] if redundant: if len(redundant) > 4: redundant = redundant[:3] + ['and %d other redundant bans' % (len(redundant) - 3)] self.msg(', '.join(redundant)) else: self.msg('No redundant bans in ' + channel) def pm_SET(self): "Command SET, to set bans and quiets comments and expiry" if len(self.word) < 3 or self.word[2].lower() in ('ban', 'quiet', 'mode', 'access', 'flag', 'config') and \ len(self.word) < 4: self.msg('wrongsyntax') return cmd = self.word[2].lower() if cmd in ('ban', 'quiet'): if len(self.word) < 4: self.msg('wrongsyntax') return channel = irclower(self.word[1]) if not channel in chanList: self.msg('idkthischan') return target = self.word[3] if not target in chanList[channel][cmd]: self.msg('bannotinlist') return args = {'user': '%(nick)s!%(ident)s@%(host)s' % self.user} params = ' '.join(self.word[4:]).rsplit('~', 1) if params[0]: args['comment'] = params[0] if len(params) == 2: expiry = mkTimedelta(params[1]) if expiry: args['expiry'] = str(expiry) chanList[channel][cmd][target].update(args) self.msg('done') def pm_EVAL(self): "apply a eval(...) in this script and return the result, for debug and development propose" if self.user['host'] != 'wikipedia/danilomac': self.msg('you are not allowed to perform this operation') return try: resp = repr(eval(' '.join(self.word[1:]))) except Exception as e: resp = repr(e) if type(resp) == unicode: resp = resp.encode('utf-8') self.msg(resp[:545]) def pm_LOGIN(self): "command to login to web interface" try: with open('login_tokens') as f: tokens = [i for i in [line.split('\t') for line in f.read().split('\n') if line] if int(i[2]) > time.time()] except IOError: self.msg('loginerror') return if not 'account' in self.user and self.runCount == 1: addhook('whois ' + self.user['nick'], self.run) bot.sendLine('WHOIS ' + self.user['nick']) return if not 'o' in chanList['#wikimedia-ops']['access'].get(self.user.get('account', ''), {}).get('flags', ''): self.msg('loginnotauthorized') return if len(self.word) < 2: self.msg('loginurl') return for i, item in enumerate(tokens): if item[0] == self.word[1]: if int(item[2]) < time.time(): self.msg('loginexpirated') return tokens[i] = [item[0], self.user['account'], item[2]] with open('login_tokens', 'w') as f: f.write('\n'.join('\t'.join(i) for i in tokens)) self.msg('logindone') with open('login.log', 'a') as f: f.write('%s\t%s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), self.user['account'])) return else: self.msg('loginfailed') def pm_LANG(self): "Command to add or modify language messages" opsFlags = chanList['#wikimedia-ops']['access'].get(self.user['account'], {}).get('flags', '') if not 'o' in opsFlags: self.msg('onlywmops') return if len(self.word) < 4 or len(self.word[1]) != 2: self.msg('wrongsyntax') return account = self.user['account'] key = '%s.%s' % (self.word[1], self.word[2]) message = ' '.join(self.word[3:]) # key (30), message (255), user (16), timestamp db.execute('''INSERT INTO lang VALUES (?, ?, ?, NOW()) ON DUPLICATE KEY UPDATE lg_message=?, lg_user=?, lg_timestamp=NOW()''', (key, message, account, message, account)) with open('lang.log', 'a') as f: f.write('%s\t%s\t[%s %s]\n' % (key, message, account, time.strftime('%Y-%m-%d %H:%M:%S'))) msgLang.setdefault(self.word[2], {})[self.word[1]] = message self.msg('langmsgsaved') def wmop_JOIN(self): "Command join, join bot to channels listed in [[meta:IRC/Channels]]" if self.ispm: return if len(self.word) < 2 or self.word[1][0] != '#': self.msg('wrongsyntax') return channel = self.word[1] if channel in chanNicks: self.msg('I am already in that channel') return if not isop(self.user, '#wikimedia-ops'): self.msg('onlywmops') return action = {'user': self.user, 'channel': self.channel, 'action': '!join'} args = {'user': self.user.full, 'time': '%d' % time.time()} if len(self.word) > 2: action['comment'] = args['comment'] = ' '.join(self.word[2:]) action['target'] = channel = self.word[1] if not chanList.get(channel, {}).get('access'): if self.runCount == 1 and channel[1:].startswith(tuple(i.replace('/', '-') for i in wikimediaCloaks)): for query in ('access ' + channel, 'ban list ' + channel, 'quiet list ' + channel): if query not in bot.queryQueue: bot.query(query) addhook('access ' + channel, self.run) return self.msg('idkthischan') return chanList[channel]['config']['KEEP'] = args def joined(): self.msg('Joined') addhook('joined ' + channel, joined) bot.join(channel) logAction(action) def wmop_PART(self): "Remove bot from a channel" if self.ispm: return if len(self.word) < 2 or self.word[1][0] != '#': self.msg('wrongsyntax') return channel = self.word[1] if not channel in chanNicks: self.msg('I am not in that channel') return if not isop(self.user, '#wikimedia-ops') and not isop(self.user, channel): self.msg('Command restricted for channel ops and Wikimedia ops') return def parted(): chanList[channel]['config'].pop('KEEP', None) self.msg('Parted') addhook('part ' + channel, parted) bot.sendLine('PART ' + channel) logAction(user= self.user, channel=self.channel, action= '!part', target=channel) def wmop_RELOAD(self): "Reload the channel access, ban and eir lists" if len(self.word) < 2 or self.word[1][0] != '#': self.msg('wrongsyntax') return channel = self.word[1] if not channel in chanList: self.msg('idkthischan') return queries = ['access ' + channel, 'ban list ' + channel, 'quiet list ' + channel] if isop('eir', channel): queries.append('eir ' + channel) if len(self.word) > 2: queries, allqueries = [], queries for query in allqueries: if query.split()[0] == self.word[2]: queries.append(query) if 'ban list ' + channel in queries: def reloadbans(): chanBans[channel] = [(b, m) for b, m in ((ban, maskre(ban)) for ban in chanList[channel]['ban']) if m] addhook('ban list ' + channel, reloadbans) for query in queries: if query not in bot.queryQueue: bot.query(query) if queries: self.msg('Reloading %s %s list%s' % (channel, mklist(q.split()[0] for q in queries), len(queries) != 1 and 's' or '')) reactor.callLater(5, dbSync) else: self.msg('No lists to reload') def wmop_CLOAKS(self): "Show number of new cloak requests" n, dt = db.query("SELECT COUNT(*), MIN(cl_timestamp) FROM cloak WHERE cl_status IN ('new', 'approved')")[0] if n: days = (dt.now() - dt).days msg = n == 1 and 'There is 1 pending cloak request, ' or \ 'There are %d pending cloak requests, the oldest was ' % n msg += 'requested %d day%s ago' % (days, days != 1 and 's' or '') else: msg = 'No new cloak requests' self.msg(msg) def pm_CLOAK(self): "Command CLOAK. Request Wikimedia cloak" print 'Cloak request begain' if not 'account' in self.user and self.runCount == 1: addhook('whois ' + self.user['nick'], self.run) bot.sendLine('WHOIS ' + self.user['nick']) return if not 'account' in self.user: self.msg('You must be registered and identified to request a cloak') return if self.user.account in [i[0] for i in db.query('SELECT cl_ircacc FROM cloak WHERE cl_status = "new"')]: self.msg('You have already requested a cloak, please wait the request be processed') return with open('.oauth.key') as f: consumer_key, consumer_secret = f.read().rstrip('\n').split('\t') url = 'https://meta.wikimedia.org/w/index.php' oauth = OAuth1(consumer_key, consumer_secret) r = requests.post(url=url, params={'title': 'Special:OAuth/initiate', 'oauth_callback': 'oob'}, auth=oauth) t = r.content.startswith('oauth_token') and dict(i.split('=', 1) for i in r.content.split('&')) if not t: with open('oauth.log', 'a') as f: f.write('%s\tError: %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), r.content)) with open('cloak.ctrl', 'a') as f: f.write('%s\t%s\t%d\t%s\t%s\n' % (t['oauth_token'], t['oauth_token_secret'], time.time(), self.user.full, self.user.account)) authorize = url + '?title=Special:OAuth/authorize&oauth_consumer_key=%s&oauth_token=%s' % (consumer_key, t['oauth_token']) self.msg('To request a Wikimedia cloak please folow this link, authorize the wiki to provide your indentification and fill the form: \x0312' + authorize) print 'Cloak request by', self.user.full, self.user.account or '(no account)' def pm_BOTCLOAK(self): "Command to confirm bot account on bot cloak requests" if len(self.word) < 2: self.msg('Wrong syntax, check the command, if the problem persists ask help in #wikimedia-opbot') if not 'account' in self.user and self.runCount == 1: addhook('whois ' + self.user['nick'], self.run) bot.sendLine('WHOIS ' + self.user['nick']) return if not 'account' in self.user: self.msg('You must be registered and identified to request a cloak') return with open('cloak.ctrl', 'r+') as f: lines = f.readlines() for i, line in enumerate(lines): if line.startswith(self.word[1]): columns = line.rstrip('\n').split('\t') if len(columns) == 6: columns.append(self.user.account) lines[i] = '\t'.join(columns) + '\n' break else: self.msg(len(columns) > 6 and 'The bot account has already been registered' or 'Internal error') return else: self.msg('There is no cloak request with that code, check the command, if problem persists ask help in #wikimedia-opbot') return f.seek(0) f.writelines(lines) self.msg('Done, now you can continue with the cloak request') def pm_PING(self): "Congigure ping on warnings" if not self.user.account: return if not any(1 for chan in chanList if 'o' in chanList[chan]['access'].get(self.user.account, {}).get('flags', '').lower()): self.msg('You are not op in any channel I know, I can\'t configure ping') return if len(self.word) == 1: self.msg('ping is set to \x02%s\x02 to you' % chanList['global']['ping'].get(self.user.account, {}).get('flags', 'off')) elif self.word[1] == 'on': chanList['global']['ping'][self.user.account] = {'flags': '+bgstw', 'time': '%d' % time.time()} self.msg('ping was set to \x02+bgstw\x02 (all pings enabled)') elif self.word[1] == 'off': if self.user.account in chanList['global'].get('ping', {}): del chanList['global']['ping'][self.user.account] self.msg('ping was disabled') elif self.word[1][0] in '-+' and len(self.word[1]) > 1: flags = set(chanList['global']['ping'].get(self.user.account, {}).get('flags', '')[1:]) sign = self.word[1][0] for c in self.word[1][1:]: if c in '+-': sign = c elif c in 'bgstwn': if sign == '+' and not c in flags: flags.add(c) elif sign == '-' and c in flags: flags.remove(c) else: self.msg('the flag "%s" don\'t exist' % c) return if flags: flags = '+' + ''.join(sorted(flags)) chanList['global']['ping'][self.user.account] = {'flags': flags, 'time': '%d' % time.time()} self.msg('ping was set to \x02%s\x02' % flags) else: if self.user.account in chanList['global'].get('ping', {}): del chanList['global']['ping'][self.user.account] self.msg('ping was disabled') else: self.msg('Syntax: ping [on|off|<+->]') def wmop_PING(self): "Returns pong" if len(self.word) > 1 and self.word[1:] == ['voice', 'notice']: self.msg('pong (only to voiced users)', prefix='+', notice='True') elif len(self.word) > 1 and self.word[1] == 'voice': self.msg('pong (only to voiced users)', prefix='+') elif len(self.word) > 1 and self.word[1:] == ['op', 'notice']: self.msg('pong (only to opped users)', prefix='@', notice='True') elif len(self.word) > 1 and self.word[1] == 'op': self.msg('pong (only to opped users)', prefix='@') else: self.msg('pong') def wmop_CHECKMODE(self): "Command to verify which channels are with modes like +r, +m, etc" now = int (time.time()) modes = [] for chan in chanList: for mode in chanList[chan]['mode']: diff = now - int(chanList[chan]['mode'][mode].get('time', 0)) if diff < 86400: modes.append('%s is %s for %s' % (chan, mode, (diff > 3600 and '%d h' % (diff / 3600) or diff > 60 and '%d min' % (diff / 60) or '%d sec' % diff))) self.msg(modes and mklist(modes) or 'No relevant channel modes set in last 24 hours') def wmop_BL(self): "Command to manage the blacklist" if len(self.word) < 2 or not self.word[1] in ('add', 'del'): self.msg('wrongsyntax') return if self.word[1] == 'add': if len(self.word) < 3: self.msg('wrongsyntax') return term = ' '.join(self.word[2:]) if term[0] == term[-1] == '/' and '|' in term: self.msg('terms with regex cannot contain |') return if term in chanList['global'].get('black', ()): self.msg('That term is already in the blacklist') return chanList['global'].setdefault('black', {})[term] = {'user': self.user.full, 'time': '%d' % time.time()} warns.loadbl() self.msg('Term added to the black list') elif self.word[1] == 'del': if len(self.word) < 3: self.msg('wrongsyntax') return term = ' '.join(self.word[2:]) if not term in chanList['global'].get('black', ()): self.msg('That term is not in the blacklist') return del chanList['global']['black'][term] warns.loadbl() self.msg('Term removed from the blacklist') wmop_BLACKLIST = wmop_BL def wmop_BOTLIST(self): "Command to manege the bot list" self.setlist('bot') def wmop_EXEMPT(self): "Command to manege the exempt list" self.setlist('exempt') wmop_WL = wmop_EXEMPT def setlist(self, _type): "Add and del itens from global lists" if not _type in chanList['global']: chanList['global'][_type] = {} if self.word[1] == 'add' or len(self.word) == 2: target = self.word[len(self.word) == 2 and 1 or 2] if not '!' in target and target[:3] != '$a:': target += '!*@*' comment = len(self.word) > 3 and ' '.join(self.word[3:]) if target in chanList['global'][_type] and not comment: self.msg('%s is already in the %s list' % (target, _type)) return chanList['global'][_type][target] = {'user': self.user.full, 'time': '%d' % time.time()} if comment: chanList['global'][_type][target]['comment'] = comment loadGLists() self.msg('%s added to the %s list' % (target, _type)) elif self.word[1] == 'del': if len(self.word) < 3: self.msg('wrongsyntax') return target = self.word[2] if not '!' in target and target[:3] != '$a:': target += '!*@*' if not target in chanList['global'][_type]: self.msg('%s is not in the %s list' % (target, _type)) return del chanList['global'][_type][target] loadGLists() self.msg('%s removed from the %s list' % (target, _type)) else: self.msg('wrongsyntax') return def wmop_STATS(self): "Report bot stats" if not self.ispm and not self.channel in ('#wikimedia-ops', '#wikimedia-ops-internal', '#wikimedia-opbot'): return nicks = {n for c in chanNicks for n in chanNicks[c]} nreg = sum(1 for n in nicks if n in nickUser and nickUser[n].account) self.msg('I am in %d channels and I see %d users, %d identified' % (len(chanNicks), len(nicks), nreg)) def wmop_LIST(self): "Return filtred list of users in channels" resp = queryBotDB(parsecmd(' '.join(self.word), not self.ispm and self.channel or None)) if isinstance(resp, basestring): self.msg(resp) return nicks = sorted(user.nick for user in resp) if not nicks: self.msg('there are no users that match these criteria') return if len(nicks) > 22: nicks = nicks[0:20] + ['%d other users' % (len(nicks) - 20)] prefixed = False if not self.ispm: for i, nick in enumerate(nicks): - if nick in chanNicks[self.channel]: + if nick in chanNicks[self.channel] and nick != 'wmopbot': nicks[i] = nick[0] + '_' + nick[1:] prefixed = True self.msg(mklist(nicks) + (prefixed and ' (_ added to avoid ping)' or '')) wmop_WHICH = wmop_LIST def wmop_WHO(self): if len(self.word) == 3 and self.word[1] == 'is': self.word[0:2] = ['whois'] return self.wmop_WHOIS() if len(self.word) == 3 and self.word[1] == 'was': self.word[0:2] = ['whowas'] return self.wmop_WHOWAS() if self.word[1] == 'banned': if len(self.word) != 5 or self.word[3] != 'in' or self.word[4][0] != '#': self.msg('wrongsintax') self.msg(whobanned(self.word[2], self.word[4])) return return self.wmop_LIST() def wmop_WARNS(self): "Enable and disable warns" if self.ispm: return if len(self.word) < 2: self.msg('wrongsyntax') return if self.word[1] == 'on': if not self.channel in warns.chanDisabled: self.msg('Warns are already enabled for all channels') return if len(self.word) > 2: chans = warns.chanDisabled[self.channel] & {x.strip(',') for x in self.word[2:]} if not chans: self.msg('Warns are already enabled for these channels') return warns.chanDisabled[self.channel] -= chans if not warns.chanDisabled[self.channel]: warns.chanDisabled.pop(self.channel) self.msg('Warns enabled for ' + mklist(chans)) elif self.channel in warns.chanDisabled: disabled = warns.chanDisabled.pop(self.channel) self.msg('Warns %senabled' % (not 'all' in disabled and 'for %s re-' % mklist(disabled) or '')) else: self.msg('Warns are already enabled') elif self.word[1] == 'off': if len(self.word) > 2 and not 'all' in warns.chanDisabled.get(self.channel, ()): warns.chanDisabled.setdefault(self.channel, set()).update(x.strip(',') for x in self.word[2:]) self.msg('Warns disabled for ' + mklist(warns.chanDisabled[self.channel])) elif not 'all' in warns.chanDisabled.get(self.channel, ()): warns.chanDisabled[self.channel] = {'all'} self.msg('Warns disabled') else: self.msg('Warns are already disabled') elif self.word[1] == 'info': status = warns.chanDisabled.get(self.channel) self.msg('Warns are ' + (not status and 'enabled' or 'disabled for ' + mklist(status))) else: self.msg('wrongsyntax') def wmop_TRACK(self): "Track users" self.setlist('track') def wmop_WHOIS(self): "Query WHOIS, NS INFO and IP whois data" if len(self.word) < 2: self.msg('wrongsyntax') return if re.match(r'\d+\.\d+\.\d+\.\d+|[\dA-Fa-f]+:[\dA-Fa-f:]+', self.word[1]): ip = self.word[1] doamin = None elif re.match(r'hex:[0-9A-Fa-f]{8}$', self.word[1]): ip = '.'.join(str(int(self.word[1][x:x + 2], 16)) for x in (4, 6, 8, 10)) doamin = None elif re.match(r'[A-Za-z-]+\.[A-Za-z]+', self.word[1]): ip = None domain = self.word[1].lower() else: user = findUser(self.word[1]) if not user: self.msg('idkthisnick') return self.whois(user) return target = ip or domain def resp(): if target in _whois: self.msg((not ip and domain and domain + ' ' or '') + 'IP %(query)s (%(city)s, %(regionName)s, %(country)s, ISP: %(isp)s)' % _whois[target]) else: self.msg('IP whois failed') sys.stderr.write('IP whois for %s failed' % self.word[1]) if target in _whois: self.msg((not ip and domain and domain + ' ' or '') + 'IP %(query)s (%(city)s, %(regionName)s, %(country)s, ISP: %(isp)s)' % _whois[target]) else: addhook('IP whois ' + target, resp) thread.start_new_thread(ipwhois, (target,)) def whois(self, user): "Response to a whois query" msg = [user.full] op = None if user.account: msg.append('account: ' + user.account) wm = tuple('#' + i[:-1] for i in wikimediaCloaks) op = [chan for chan in chanList if chan.startswith(wm) and user.account in chanList[chan]['access'] and 'o' in chanList[chan]['access'][user.account].get('flags', '').lower()] if 'reg' in user: t = int(time.time()) - int(user.reg) delta = t / 31557600 and (t / 31557600, 'year') or t / 2629800 and (t / 2629800, 'month') or \ (t / 86400, 'day') msg.append('registered %d %s%s ago' % (delta[0], delta[1], delta[0] != 1 and 's' or '')) if op: op = sorted(op, key=lambda c: c in chanNicks and len(chanNicks[c]), reverse=True) msg.append('is op in ' + mklist(len(op) <= 7 and op or op[:5] + ['%d more channels' % (len(op) - 5)])) if not op: realname = user.get('realname', '') msg.append('realname: ' + (len(realname) > 50 and realname[:45] + '...' or realname)) t = user.talked and user.talked.isdigit() and (int(time.time()) - int(user.talked)) / 60 if t: t = t >= 60 and (t / 60, 'hour') or (t, 'minute') t = 'talked %d %s%s ago' % (t[0], t[1], t[0] != 1 and 's' or '') msg.append(t or user.talked and 'talked' or 'hasn\'t even spoken') msg.append('score %d' % user.score) if 'ip' in user: msg.append('IP %s (country: %s, ISP: %s)' % (user.ip, user.get('country', ''), user.get('isp', ''))) if not user.host.startswith(wikimediaCloaks): chans = [chan for chan in chanNicks if user.nick in chanNicks[chan]] msg.append('is now in ' + mklist(len(chans) > 5 and chans[:4] + ['%d more channels' % (len(chans) - 4)] or chans or ['no channel'])) self.msg(', '.join(msg)) def wmop_HELP(self): "Return the documentation page" self.msg('main commands: !join #channel; !part #channel; !blocks nick|IP|#channel; !checkban nick|mask|#channel;\ !redundant #channel; !reload #channel; !where nick; !list #channel filters; !whois nick|IP|domain; !botlist add botmask;\ !exempt add mask; !track mask; \x0312https://tools.wmflabs.org/wmopbot/help') def cmd_STATUS(self): "Set a new status in the channel topic" channel = self.channel if not channel in ('#wikimedia-opbot', '#wikimedia-cloud'): return topic = topics[channel] new = re.sub(r'(?<=\| [Ss]tatus: )[^\|]*?(?= \|)', ' '.join(self.word[1:]), topic) if new == topic: bot.msg(channel, 'No changes to apply in the status') else: bot.sendLine('CS TOPIC %s %s' % (channel, new)) def mkmode(cmdlist, channel): modes, args = '', [] commands = [] count = 0 while cmdlist: m, arg = cmdlist.pop(0) if not modes or m[0] != sign: modes += m sign = m[0] else: modes += m[1] count += 1 if arg: args.append(arg) if count == 4: commands.append('MODE %s %s %s' % (channel, modes, ' '.join(args))) count, modes, args = 0, '', [] if modes: commands.append('MODE %s %s %s' % (channel, modes, ' '.join(args))) return commands def checkban(ban): "For security, check for *!*@* ban" if re.match(r'^\*?!\*?@\*?$', ban) or not '!' in ban or not '@' in ban: print '[%s] Error: tried to ban %s' % (time.strftime('%Y-%m-%d %H:%M:%S'), ban) return False return True configFlags = {'GLOBAL': 0, 'ASKCOMM': 0, 'RMBAM': 1, 'RMQUIET': 1, 'OPLOG': 0, 'BOTSAY': 0, 'WM': 0} reIP = re.compile(r'\d{1,3}[.-]\d{1,3}[.-]\d{1,3}[.-]\d{1,3}|[0-9a-f]{1,4}(?:::?[0-9a-f]{1,4}){1,8}(?:::)?', re.I) def ip2numcdir(ip): "transforms an IP string into IP range tuple, e.g. '1.2.3.4/30' => (16909060, 30)" ipv4 = re.match(r'(\d{1,3}\.(?:\*|\d{1,3})\.(?:\*|\d{1,3})\.(?:\*|\d{1,3}))(?:/([1-3]?\d))?', ip) if ipv4: octets = ipv4.group(1).split('.') cdir = ipv4.group(2) and int(ipv4.group(2)) or 32 if cdir > 32: cdir = 32 for i, octet in enumerate(octets): if octet == '*': _cdir = i * 8 cdir = _cdir if not cdir or cdir > _cdir else cdir octets[i] = 0 else: octet = int(octet) octets[i] = octet if octet < 256 else 255 ip = reduce(lambda a, b: a << 8 | b, octets) return (ip, cdir) ipv6 = re.match(r'([0-9a-fA-F]{1,4}(?:::?[0-9a-fA-F]{1,4}){1,7}(?:::)?)(?:/(1?\d{1,2}))?', ip) if ipv6: parts = [[g or '0' for g in part.split(':')] for part in ipv6.group(1).split('::')] groups = parts[0] + (len(parts) == 2 and ['0'] * (8 - len(parts[0]) - len(parts[1])) + parts[1] or []) ip = reduce(lambda a, b: a << 16 | b, [int(g, 16) for g in groups]) cdir = ipv6.group(2) and int(ipv6.group(2)) or 128 return (ip, cdir) else: return False def num2ip(ip, cdir=None): "Tranforms an IP tuple into a IP format string; e.g. (16909060, 30) => '1.2.3.4/30'" # IPv6 if ip > 4294967295: if cdir == None or cdir > 128: cdir = 128 if ip < 128: ip = ip >> 128 - cdir << 128 - cdir ipv6 = ':'.join('%x' % (ip >> 112 - i * 16 & 65535) for i in range(8)) return re.sub(':0(?::0)*(?::|$)', '::', ipv6, 1) + ('/%d' % cdir if cdir < 128 else '') # IPv4 if cdir == None or cdir > 32: cdir = 32 ipv4 = ip >> 32 - cdir << 32 - cdir return '.'.join(str(ip >> 24 - i * 8 & 255) for i in range(4)) + ('/%d' % cdir if cdir < 32 else '') def testip(A, B): "Test if IP or IP range A fits in IP range B" ipA, cdirA = type(A) == tuple and A or ip2numcdir(A) ipB, cdirB = type(B) == tuple and B or ip2numcdir(B) if ipA > 4294967295 and cdirB <= cdirA and ipA >> 128 - cdirB == ipB >> 128 - cdirB: return True if ipA < 4294967295 and cdirB <= cdirA and ipA >> 32 - cdirB == ipB >> 32 - cdirB: return True return False WikimediaIPs = [ip2numcdir(ip) for ip in ('2620:0:860::/46', '198.35.26.0/23', '208.80.152.0/22')] if not 'chanBans' in globals(): chanBans = {chan: [(b, m) for b, m in ((ban, maskre(ban)) for ban in chanList[chan]['ban']) if m] for chan in chanList if len(chanList[chan]['ban']) > 1} def redundantBans2(): "generate a list of redundant bans for all channels" redundant = [] for channel in sorted(chanBans): check = [channel] + [c for c in jBans if channel in jBans[c] and c in chanBans] redundant.extend([(channel, ban2check, ban + (checkChan != channel and ' in ' + checkChan or '')) for checkChan in check for ban, mask in chanBans[checkChan] for ban2check in chanList[channel]['ban'] if (checkChan != channel if ban == ban2check else mask.match(ban2check))]) with open('redundant-bans.txt', 'w') as f: f.write('\n'.join('{}: {} is already covered by {}'.format(*r) for r in redundant)) csTemp = None reCSLine = re.compile(r'\d+ +(\S+) +\+(\w+)(?: \(([\w-]+)\))? \[modified ([?\w ]+?) ago\]') def csNotice(msg): global csTemp with open('csnotices.log', 'a') as f: f.write('%s\n' % msg) if msg.startswith(('Information on', 'Entry Nickname/Host')): csTemp = [msg] elif msg == '\x02*** End of Info ***\x02': channel = irclower(csTemp[0].split('\x02')[1]) for line in csTemp: if line.startswith('Registered :'): regtime = parseTime(line) if regtime: chanList[channel]['config'].setdefault('cs/irc', {})['registered'] = str(regtime) elif line.startswith('Mode lock :'): chanList[channel]['config'].setdefault('cs/irc', {})['mlock'] = line.split(':')[1].strip() elif line.startswith('Flags :'): chanList[channel]['config'].setdefault('cs/irc', {})['csflags'] = line.split(':')[1].strip() chanList[channel]['config'].setdefault('update', {})['csinfo'] = str(int(time.time())) csTemp = None callhook('csinfo ' + channel) elif msg.startswith('End of \x02#'): channel = irclower(msg.split('\x02')[1]) for line in csTemp: match = reCSLine.match(line) if match: entry, flags, template, timedelta = match.groups() else: continue if channel == '#wikimedia-ops': if not 'nonwm' in locals(): nonwm = {u['account'] for u in nickUser.itervalues() if 'acount' in u and not u['host'].startswith(wikimediaCloaks)} if 'o' in flags and entry in nonwm and not entry in chanList['global'].get('flag', {}).get(entry): chanList['global'].setdefault('flags', {})[entry] = {'flags': 'WM', 'user': 'wmopbot', 'comment': 'automaticaly added WM flag: user has not wm cloak but is op in #wikimedia-ops', 'time': '%d' % time.time(), 'change': '+WM'} chanList[channel]['access'].setdefault(entry, {}).update({'flags': flags, 'time': csTime(timedelta)}) if template: chanList[channel]['access'][entry]['template'] = template csTemp = None chanList[channel]['config'].setdefault('update', {})['access'] = str(int(time.time())) callhook('access ' + channel) elif msg.endswith('\x02 is not registered.'): channel = irclower(msg.split('\x02')[1]) chanList[channel]['config'].setdefault('update', {})['access'] = str(int(time.time())) chanList[channel]['config']['update']['csinfo'] = str(int(time.time())) chanList[channel]['config']['update']['cserror'] = 'not registered' callhook('not registered ' + channel) elif msg == 'You are not authorized to perform this operation.': if bot.queryRunning and bot.queryRunning[1].startswith(('access', 'csinfo')): channel = bot.queryRunning[1].split()[1] chanList[channel]['config'].setdefault('update', {})['access'] = str(int(time.time())) chanList[channel]['config']['update']['csinfo'] = str(int(time.time())) chanList[channel]['config']['update']['cserror'] = 'not authorized' callhook('cs not authorized') elif csTemp: csTemp.append(msg) else: print '[%s] ChanServ unknown notice: %s' % (time.strftime('%Y-%m-%d %H:%M:%S'), msg) def csTime(timedelta): if '?' in timedelta: return '?' seconds = {'y': 31536000, 'w': 604800, 'd': 86400, 'h': 3600, 'm': 60, 's': 1} oldtime = int(time.time()) for delta in re.findall(r'\d+[ywdhms]', timedelta): oldtime -= int(delta[:-1]) * seconds[delta[-1]] return str(oldtime) nsTemp = None def nsNotice(msg): global nsTemp with open('nsnotices.log', 'a') as f: f.write('%s\n' % msg) if msg.startswith(('Information on')): nsTemp = msg.split('\x02')[1::2][-1] if nsTemp in nsQueue: nsQueue.remove(nsTemp) if nsQueue and lastNsInfo + 10 <= time.time(): reactor.callLater(10, bot.sendLine, (nsQueue[0],)) lastNsInfo = time.time() elif msg == '\x02*** End of Info ***\x02': nsTemp = None elif nsTemp and msg.startswith(('Registered', 'User reg.')): m = re.search(r': ([\w: ]+) \(', msg) reg = int(time.mktime(time.strptime(m.group(1), '%b %d %H:%M:%S %Y'))) for user in nickUsers.itervalues(): if user.account == nsTemp and (not 'reg' in user or int(user['reg']) < reg): user['reg'] = str(reg) eirChan = None eirComment = {} def eirNotice(msg): "Called when I receive an notice from eir (bantracker bot)" global eirChan if msg == 'Done': return if msg == 'No results': callhook('eir no results') return if msg == 'End of results' and eirChan: callhook('eir ' + eirChan) eirChan = None return if msg[0] != '\x02': return data = msg.split('\x02')[1::2] if not 6 <= len(data) <= 7: print '[%s] unknown eir notice: %s' % (time.strftime('%Y-%m-%d %H:%M:%S'), msg) return # data example: ['ban', '*!*@unaffiliated/geo23', '#wikimedia-ops', 'notaspy!~nick@wikimedia/nick', '2014-10-09 15:48:04Z', 'Geo23', '4752-09-04 15:50:15Z.'] if len(data) == 6: _type, target, channel, user, _time, expiry = data comment = None else: _type, target, channel, user, _time, comment, expiry = data eirChan = channel expiry = eirTime(expiry) if channel in chanList and target in chanList[channel].get(_type, {}) and ('eir' in chanList[channel][_type][target] or not chanList[channel][_type][target].get('comment')): d = {'time': str(eirTime(_time)), 'eir': 'yes', 'expiry': expiry > 31536000 + time.time() and 'p' or str(expiry)} if comment: d['comment'] = comment if chanList[channel][_type][target].get('user', '?') == '?': d['user'] = user chanList[channel][_type][target].update(d) def eirTime(txt): match = re.match(r'(\d+)-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', txt) if int(match.group(1)) > 3000: # some years like '29396-02-10 21:56:39' can not be parsed by time.strptime txt = '3000' + match.group(0)[len(match.group(1)):] else: txt = match.group(0) return int(time.mktime(time.strptime(txt, '%Y-%m-%d %H:%M:%S'))) def chanMsg(user, channel, msg): "Called when I receive a channel message from an non bot user" hourChanStats[int(time.time()) / 3600][channel] += 1 user['talked'] = '%d' % time.time() def pmNotice(user, message): "Called when I receive an private notice that is not from ChanServ" if user.full == 'eir!eir@freenode/utility-bot/eir': eirNotice(message) elif user.full == 'NickServ!NickServ@services.': nsNotice(message) else: print 'NOTICE from %s: %s' % (user['nick'], message) def botPM(user, msg): "Called when I receive a private message from a bot" if user.full == 'eir!eir@freenode/utility-bot/eir' and msg.startswith('Please comment on the following:'): print '[%s] eir: %r' % (time.strftime('%Y-%m-%d %H:%M:%S'), msg) ask = re.search(' (ban|quiet)\[(\d+)\] (\S+) ', msg.replace('\x02', '')) action, _id = ask and ((ask.group(1), ask.group(3)), ask.group(2)) or (None, None) comment, expiry = ask and eirComment.get(action, (None, None)) if comment: del eirComment[action] bot.msg('eir', comment or 'no comment') if expiry and _id: bot.msg('eir', 'btset %s ~%s' % (_id, expiry == 'p' and '3652d' or expiry)) return print 'bot pm', user.full, msg if user.host == 'wikimedia/bot/wm-bot': ops = re.match(r'OPS (\S+) in (\S+): !ops(.*)', msg) if not ops: return channel = irclower(ops.group(2)) if channel in chanNicks or channel in calledops: return user = User(ops.group(1)) logAction(user=user, channel='#wikimedia-ops', action='!ops', target=channel) def callops(): calledops.add(channel) ping = warns.mkping(channel) bot.msg('#wikimedia-opbot,#wikimedia-ops', '%s wants ops attention in \x02%s\x02%s%s' % (user.nick, channel, ops.group(3) and ' (%s)' % ops.group(3).strip() or '', ping and ' \x0315ping ' + mklist(sorted(ping)) or '')) if channel in chanList: callops() else: print 'wm-bot !ops query access' bot.query('access ' + channel) addhook('access ' + channel, callops) def botChanMsg(user, channel, msg): "Receive channel messages sent by bots" if channel == '#mediawiki-feed' and user.nick == 'wikibugs' and '\x0310wikimedia-irc-freenode\x0f' in msg: bot.msg('#wikimedia-ops', msg) def parseTime(txt): match = re.search(r'(\w{3} \d\d \d\d:\d\d:\d\d \d{4})', txt) if match: try: return int(time.mktime(time.strptime(match.group(1), '%b %d %H:%M:%S %Y'))) except: return def initialized(): "Called whe the bot is initialized" modechans = [(chan, {m[1] for m in chanList[chan]['mode']}) for chan in chanList if chanList[chan]['mode']] for chan, mode in modechans: if 'q' in mode: bot.query('quiet list ' + chan) if 'b' in mode: bot.query('ban list ' + chan) if mode - {'q', 'b'}: bot.query('mode ' + chan) if not 'lastCheck' in globals(): lastCheck = 0 if not 'queryEir' in globals(): queryEir = set() def periodic(): "Periodic checks" start = time.time() now = int(time.time()) remove = set() with open('.online', 'w') as f: f.write('%d\n%s' % (time.time(), '\n'.join(chanNicks))) for chan in chanList: for mode in ('ban', 'quiet'): for target in chanList[chan][mode].keys(): if 'eir' in chanList[chan][mode][target]: continue # don't need to remove bans tracked by eir expiry = chanList[chan][mode][target].get('expiry') if expiry and expiry != 'p' and int(expiry) < now: remove.add(chan) for chan in remove: bot.remove(chan) for chan in list(queryEir): bot.query('eir ' + chan) queryEir.remove(chan) for chan in waitingSupport: if waitingSupport[chan]['time'] + 1800 < now: del waitingSupport[chan] for u in warns.recentActions.keys(): if warns.recentActions[u][0][2] + 600 > now: del warns.recentActions[u] if len(hourChanStats) > 1: hour = min(hourChanStats) stats = hourChanStats.pop(hour) db.execute('INSERT INTO stats VALUES %s' % ','.join(['(?,?,?,?)'] * len(stats)), tuple(i for c in stats for i in (c, hour, len(chanNicks.get(c, ())), stats[c]))) calledops.clear() if not bot.main.whoisQueue: bot.main.whoisQueue.extend([n for n in nickUser if 'unreg' in nickUser[n] and int(nickUser[n]['unreg']) + 86400 < time.time()][0:5]) if bot.main.whoisQueue: bot.main.sendLine('WHOIS ' + bot.main.whoisQueue[0]) # hourly checks global lastCheck, chanBans if lastCheck + 3500 > now: return api = urlopen('https://meta.wikimedia.org/w/api.php?action=query&format=json&prop=revisions&titles=IRC/Channels&rvprop=content') data = json.loads(api.read()) text = data['query']['pages']['6237']['revisions'][0]['*'] metaChannels = {'#' + str(chan).lower() for chan in re.findall(r'\n\| ?\{\{[Cc]hannel\|([^ |}]+)\}\}', text)} day = now - 86400 count = 0 metaChanges = [[], []] for chan in metaChannels: if not chan in chanList or not 'WM' in chanList[chan]['config']: chanList[chan]['config']['WM'] = {'time': '%d' % time.time()} metaChanges[0].append(chan) print 'meta add ' + chan for chan in chanList: if not 'WM' in chanList[chan]['config'] and not 'KEEP' in chanList[chan]['config']: continue if 'WM' in chanList[chan]['config'] and not chan in metaChannels: del chanList[chan]['config']['WM'] metaChanges[1].append(chan) print 'meta del ' + chan continue count += 1 check = chanList[chan]['config'].get('update', {}) if not 'ban' in check or int(check['ban']) < day: bot.query('ban list ' + chan) if isop('eir', chan): bot.query('eir ' + chan) if not 'quiet' in check or int(check['quiet']) < day: bot.query('quiet list ' + chan) if not 'mode' in check or int(check['mode']) < day: bot.query('mode ' + chan) if not 'access' in check or int(check['access']) < day: bot.query('access ' + chan) if not 'csinfo' in check or int(check['csinfo']) < day: bot.query('csinfo ' + chan) if len(bot.queryQueue) > 200: break if any(metaChanges): madd = len(metaChanges[0]) > 10 and metaChanges[0][0:8] + ['%d more channels' % (len(metaChanges[0]) - 8)] \ or metaChanges[0] mdel = len(metaChanges[1]) > 10 and metaChanges[1][0:8] + ['%d more channels' % (len(metaChanges[1]) - 8)] \ or metaChanges[1] msg = (madd and 'added channels: %s' + mklist(madd) or '') + (madd and mdel and '; ' or '') + \ (mdel and 'removed channels: %s' + mklist(mdel) or '') bot.msg('#wikimedia-opbot', 'recent changes in [[meta:IRC/Channels]]: ' + msg) lastCheck = now ipBlocks.clear() del chanBans chanBans = {chan: [(b, m) for b, m in ((ban, maskre(ban)) for ban in chanList[chan]['ban']) if m] for chan in chanList if len(chanList[chan]['ban']) > 1} warn = [] for chan in chanList: for mode, args in chanList[chan]['mode'].iteritems(): if mode[1] in 'rmbq' and (now - int(args.get('time', 0))) / 3600 in (2, 4, 8, 16, 24) \ and args.get('user') != 'Sigyn!sigyn@freenode/utility-bot/sigyn': warn.append((chan, mode, (now - int(args.get('time', 0))) / 3600)) if warn: ping = {n for c, m, h in warn for n in warns.mkping(c, 'ts') if h == 2} opbotmsg = ', '.join('%s is %s for more than %d hours' % (c, m, h) for c, m, h in warn) + \ (ping and ' \x0315ping ' + ' '.join(sorted(ping)) or '') if any('#wikimedia-opbot' in i[0] for i in warn): warn = [i for i in warn if not '#wikimedia-opbot' in i[0]] ping = {n for c, m, h in warn for n in warns.mkping(c, 'ts') if h == 2} opsmsg = ', '.join('%s is %s for more than %d hours' % (c, m, h) for c, m, h in warn) + \ (ping and ' \x0315ping ' + ' '.join(sorted(ping)) or '') if opsmsg: bot.msg('#wikimedia-ops', opsmsg) bot.msg('#wikimedia-opbot', opbotmsg) elif opbotmsg: bot.msg('#wikimedia-ops,#wikimedia-opbot', opbotmsg) _whois.clear() for chan in chanNicks: if 'Sigyn' in chanNicks[chan] and not 'Sigyn' in chanList[chan]['config']: chanList[chan]['config']['Sigyn'] = {'since': '%d' % time.time()} elif 'Sigyn' in chanList[chan]['config'] and not 'Sigyn' in chanNicks[chan]: del chanList[chan]['config']['Sigyn'] def modeUser(user, channel, mode, args): "Called when I see an user setting mode +b, -b, +q, -q, +e, -e, +I or -I" if mode == '+b': banned(user, channel, args) elif mode == '-b': unbaned(user, channel, args) elif mode == '+q': banned(user, channel, args, 'quiet') elif mode == '-q': unbaned(user, channel, args, 'quiet') def banned(user, channel, target, _type='ban'): "Called when I see a ban or quiet action" if target == '$~a': chanMode(user, channel, '+%s $~a' % _type[0]) # I banned if user.nick in bot.nicks: print 'I banned' for cmd in opCommands: if cmd.get(_type) == target and cmd['channel'] == channel: comment, expiry, trigged = cmd.get('comment', ''), cmd.get('expiry', ''), cmd.get('user', '') print 'banned opCmomands', cmd break else: print 'banned no opCommand' cmd = {'user': 'wmopbot', 'channel': channel, action: _type, _type: target} comment, expiry, trigged = None, None, None logAction(cmd, target=cmd[_type]) args = {'user': isinstance(trigged, dict) and trigged.full or trigged or 'wmopbot', 'time': '%d' % time.time()} if not comment and not expiry and isinstance(trigged, dict) and 'ASKCOMM' in chanList[channel]['config']: print 'banned askComment', user, channel, _type, target askComment(cmd['user'], channel, _type, target) elif comment or expiry: if comment: args['comment'] = comment t = expiry and mkTimedelta(expiry) if t: args['expiry'] = t # Other user banned else: args = {'user': user.full, 'time': '%d' % time.time()} if 'RM' + _type.upper() in chanList[channel]['config']: args['expiry'] = mkTimedelta(chanList[channel]['config']['RM' + _type.upper()]['arg']) if 'ASKCOMM' in chanList[channel]['config']: askComment(user, channel, _type, target) elif 'eir' in chanNicks[channel]: queryEir.add(channel) logAction(args, channel=channel, action=_type, target=target) chanBans[channel] = [(b, m) for b, m in ((ban, maskre(ban)) for ban in chanList[channel]['ban']) if m] if target in chanList[channel][_type]: # delete db duplicates db.execute('DELETE FROM list WHERE (ls_channel, ls_type, ls_target) = (?,?,?)', (channel, _type, target)) chanList[channel][_type][target] = args db.execute('INSERT INTO list VALUES (?, ?, ?, ?)', (channel, _type, target, dumpArgs(args))) if _type == 'ban' and target.startswith('$j:#'): jBans.setdefault(target[3:], []).append(channel) if _type == 'ban': chanBans[channel] = [(b, m) for b, m in ((ban, maskre(ban)) for ban in chanList[channel]['ban']) if m] def unbaned(user, channel, target, _type='ban'): if target == '$~a': chanMode(user, channel, '-%s $~a' % _type[0]) "Called when I see an unban or unquiet action" params = {'user': '?', 'time': '?', 'expiry': '', 'target': target, 'channel': channel, 'action': 'un' + _type} if target in chanList[channel][_type]: params.update(chanList[channel][_type][target]) del chanList[channel][_type][target] db.execute('DELETE FROM list WHERE (ls_channel, ls_type, ls_target) = (?,?,?)', (channel, _type, target)) if params.get('user', '?') != '?': params['note'] = '%s by %s' % (_type, params['user']) logAction(params, user=user) params['user'] = user if _type == 'ban' and target.startswith('$j:#') and channel in jBans.get(target[3:], []): jBans[target[3:]].remove(channel) if _type == 'ban': chanBans[channel] = [(b, m) for b, m in ((ban, maskre(ban)) for ban in chanList[channel]['ban']) if m] message = '%s %s %s in %s' % (user.nick in bot.nicks and 'I' or user.full, {'ban': 'unbanned', 'quiet': 'unquieted'}[_type], target, channel) if 'comment' in params: message += '; comment: ' + params['comment'] if 'expiry' in params: message += params['expiry'] == 'p' and '; permanent' or params['expiry'].isdigit() and \ '; expiry: ' + time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(params['expiry']))) or '' if user.nick in bot.nicks: bot.msg('#wikimedia-opbot', message) def join(user, channel): "Called when an user join a channel" if user.ip and (not user.isp and not user.country): if user.ip in _whois: user['country'] = _whois[user.ip].get('country', '') user['isp'] = _whois[user.ip].get('isp', '') elif not user.ip in whoisQueue: whoisQueue.append(user.ip) if lastWhois + 10 < time.time(): thread.start_new_thread(ipwhois, (user.ip,)) def kicked(user, channel, target, reason): "Called when I see a kick action" logAction(user=user, channel=channel, action='kick', target=target, note=reason != target and 'reason: ' + reason or '') if target.nick in bot.nicks: bot.msg('#wikimedia-opbot', '\x01ACTION was kicked from %s by %s\x01' % (channel, user.nick)) def quit(user, channels, reason): "Called when I see a user quit IRC" if reason.startswith(('Killed', 'K-Lined')) and not reason.endswith('(Nickname regained by services))'): way = reason.split()[0] channel = ' '.join(channels)[:50] logAction(user='Sigyn' in reason and 'Sigyn' or '?', channel=channel, action=way, target=user.full, note=reason) def part(user, channel, reason=''): "Called when I see an user parting the channel" match = re.match(r'requested by (\S+)', reason) if match: logAction(user=nickUser[match.group(1)], channel=channel, action='remove', target=user.full, note=reason) def chanMode(user, channel, mode, arg=None): "Called when I see a channel mode change" if mode[0] == '-': if '+' + mode[1:] in chanList[channel]['mode']: del chanList[channel]['mode']['+' + mode[1:]] else: chanList[channel]['mode'][mode] = {'user': user.full, 'time': '%d' % time.time()} if arg: chanList[channel]['mode'][mode]['arg'] = arg if user.host == 'freenode/utility-bot/sigyn': global sigynDefcon if not sigynDefcon or not sigynDefcon.active(): sigynDefcon = reactor.callLater(1, defconMode, mode[0] == '+') return bot.msg('#wikimedia-opbot', '\x02%s\x02 was set in %s' % (mode + (arg and ' ' + arg or ''), channel)) def logAction(params={}, **extra): "Register an action" params = dict(params, **extra) try: user, channel, action, target = [params[i] for i in ('user', 'channel', 'action', 'target')] except: print 'Error while logging action: %r' % params return args = [params.get('note')] + ['comment' in params and 'comment: ' +params['comment']] + \ [params.get('time', '').isdigit() and time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(params['time'])))] \ + ['expiry' in params and (params['expiry'] == 'p' and 'permanent' or params['expiry'].isdigit() and time.strftime('expiry: %Y-%m-%d %H:%M:%S', time.gmtime(int(params['expiry'])))) or params.get('expiry')] args = '; '.join(a for a in args if a) user = isinstance(user, dict) and user.full or user[:90] target = isinstance(target, dict) and target.full or target[:90] # actions: id, user(90), chan(50), action(10), target(90), args(255), timestamp db.execute('''INSERT INTO actions (ac_user, ac_channel, ac_action, ac_target, ac_args, ac_timestamp) VALUES (?, ?, ?, ?, ?, ?)''', (user, channel, action, target, args[:255], time.strftime('%Y-%m-%d %H:%M:%S'))) def askComment(user, channel, _type, target): "Ask an user to comment your ban/quiet/mode" if not user['nick'] in commQueue: bot.msg(user['nick'], langMsg(user, 'askcomment', _type, target, channel)) commQueue[user['nick']].append((channel, _type, target)) def privateMsg(user, msg): "Called when I receive a private message from an user and it is not a command" if not user['nick'] in commQueue: if msg == 'cloak': bot.msg(user.nick, 'You need to be identified to request a cloak') print 'Cloak request from non identified user', user.full, user.account or '(no account)' else: print 'No command or comment message:', user.full, msg return nick = user['nick'] # commQueue[nick] = [(channel, _type, target), ...] channel, _type, target = commQueue[nick].pop(0) if not commQueue[nick]: del commQueue[nick] msg = msg.rsplit('~', 1) comment = msg[0].strip() if len(msg) == 2: expiry = mkTimedelta(msg[1]) elif _type == 'ban' and 'RMBAN' in chanList[channel]['config']: expiry = mkTimedelta(chanList[channel]['config']['RMBAN']['arg']) elif _type == 'quiet' and 'RMQUIET' in chanList[channel]['config']: expiry = mkTimedelta(chanList[channel]['config']['RMQUIET']['arg']) else: expiry = None if not comment and not expiry: return try: args = chanList[channel][_type][target] except Exception as e: print '[%s] Error while geting ban/quiet/mode data: nick: %s, channel: %s, type: %s, target: %s, error: %r' % \ (time.strftime('%Y-%m-%d %H:%M:%S'), nick, channel, _type, target, e) bot.msg(nick, 'internalerror') return if comment: args['comment'] = comment if expiry: args['expiry'] = str(expiry) chanList[channel][_type][target] = args db.execute('UPDATE list SET ls_args = ? WHERE ls_channel = ? AND ls_type = ? AND ls_target = ?', (dumpArgs(args), channel, _type, target)) bot.msg(nick, 'done') def mkTimedelta(expiry): "Transform timedelta (e.g. 24h) in unix timestamp" timedelta = 0 t = {'d': 86400, 'h': 3600, 'm': 60} for i in re.findall(r'\d+ ?[dhm]', expiry): timedelta += int(i[:-1]) * t[i[-1]] if timedelta: return '%d' % (time.time() + timedelta) elif expiry.strip().lower() == 'p': return 'p' else: return False def removeMode(channel, mode, target): "Remove ban, quiet and other channel modes" expiry = chanList[channel].get(mode, {}).get(target, {}).get('expiry') if not expiry or expiry == 'p' or int(expiry) > time.time(): print '[%s] tried to remove %s %s %s before expiry %s' % (time.strftime('%Y-%m-%d %H:%M:%S'), channel, mode, target, expiry == 'p' and expiry or time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(expiry)))) return nicks = {nick: status for nick, status in chanNicks[channel].items() if nick in bot.nicks} sys.stderr.write('nicks: %r' % nicks) modes = {'ban': 'b', 'quiet': 'q'} m = mode in modes and modes[mode] or mode def oped(): bot.sendLine('MODE %s -%s %s' % (channel, m, target)) if nicks and 2 in nicks.values(): oped() return elif nicks and 'o' in chanList[channel]['access'].get('wmopbot', {}).get('flags', ''): bot.sendLine('CS OP ' + channel) addhook('oped ' + channel, oped) else: bot.msg('#wikimedia-opbot', 'can not remove expired %s from %s' % (mode, channel)) ipBlocks = {} def wikiBlock(ip): "Verify wiki blocks for an IP" if ip in ipBlocks: return ipBlocks[ip] ips = [] ipv4 = re.match(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip) if ipv4: octets = [int(o) for o in ipv4.group(0).split('.')] num = reduce(lambda a, b: a << 8 | b, octets) for cdir in range(16, 33): n = num >> 32 - cdir << 32 - cdir ips.append('.'.join(str(n >> 24 - i * 8 & 255) for i in range(4)) + ('/%d' % cdir if cdir < 32 else '')) else: ipv6 = re.match(r'[0-9a-fA-F]{1,4}(?:::?[0-9a-fA-F]{1,4}){1,7}(?:::)?', ip) if not ipv4 and ipv6: parts = [[g or '0' for g in part.split(':')] for part in ipv6.group(0).split('::')] groups = parts[0] + (len(parts) == 2 and ['0'] * (8 - len(parts[0]) - len(parts[1])) + parts[1] or []) num = reduce(lambda a, b: a << 16 | b, [int(g, 16) for g in groups]) for cdir in range (22, 129): n = num >> 128 - cdir << 128 - cdir ips.append(re.sub(':0(?::0)*$', '::', ':'.join('%x' % (n >> 112 - i * 16 & 65535) for i in range(8)), 1) + \ ('/%d' % cdir if cdir < 128 else '')) if not ips: ipBlocks[ip] = False return False sql = 'SELECT wb_ip, wb_wikis FROM wikiblocks WHERE wb_ip IN (%s)' % ','.join('"%s"' % ip for ip in ips) with open('verifyblocks.log', 'a') as f: f.write(sql + '\n') r = db.query(sql) if not r: return False wikis = ['%s (%s)' % (', '.join(w.split()), i) for i, w in r] if wikis: ipBlocks[ip] = ', '.join([i for i in wikis if 'global' in i] + [i for i in wikis if not 'global' in i]) \ .replace('global', '\x02global\x02') return ipBlocks[ip] else: ipBlocks[ip] = False return False def ipwhois(target): "Query IP whois" global lastWhois if target in whoisQueue: whoisQueue.remove(target) if whoisQueue and lastWhois + 10 <= time.time(): reactor.callLater(10, thread.start_new_thread, ipwhois, (whoisQueue[0],)) lastWhois = time.time() try: api = urlopen('http://ip-api.com/json/' + target, timeout=2) r = json.loads(api.read()) except: print 'IP whois failed for', target return if r.get('status') == 'success': _whois[target] = r for user in nickUser.itervalues(): if 'ip' in user and user.ip == target: user['country'] = decode(r.get('country', '')) user['isp'] = decode(r.get('isp', '')) reactor.callFromThread(callhook, 'IP whois ' + target) def decode(txt): return txt.encode('utf-8') if isinstance(txt, unicode) else txt def isBot(user): "Verify if an user is the bot list" return isinlist(user, 'bot') def loadGLists(): "Transforms the global lists into precompiled regex lists" global globalList if not 'globalList' in globals(): globalList = {} for _type in ('bot', 'exempt', 'track'): if not _type in chanList['global']: continue globalList[_type] = {} for entry in chanList['global'][_type]: if entry[0:3] == '$a:' and len(entry) > 3: globalList[_type][entry[3:]] = chanList['global'][_type][entry].get('args') continue if '!' in entry: mask = maskre(entry) if not mask: print 'Error in loadGLists: failed to transforms %s into a regex' % entry continue globalList[_type][mask] = chanList['global'][_type][entry].get('args') loadGLists() def isinlist(user, _type): "Ckeck if the user is in a global list" if not _type in globalList: return False fulluser = user.full for entry in globalList[_type]: if user.account == entry if type(entry) == str else entry.match(fulluser): return True return False def defconMode(active): if active: bot.msg('#wikimedia-opbot', 'Sigyn defcon mode on') #def opped(): # bot.sendLine('MODE #wikimedia-opbot +qz $~x:*@*/*#*') #reactor.callLater(1, bot.sendLine, 'CS OP #wikimedia-opbot') #addhook('op #wikimedia-opbot', opped) else: bot.msg('#wikimedia-opbot', 'Sigyn defcon mode off') #bot.sendLine('MODE #wikimedia-opbot -qzo $~x:*@*/*#* wmopbot') def findUser(nick): "Case insensitive search for user nick" nick = irclower(nick) for n in nickUser: if nick == irclower(n): return nickUser[n] return False def parsecmd(msg, here=None): words = [] quote = False for word in msg.strip('?').split(): if not quote and word[0] in ('"', '\''): quote = word[0] words.append(word[1:]) elif quote: if word[-1] == quote: words[-1] += word[0:-1] else: words[-1] += word else: words.append(word.rstrip(',')) command = {} prev = '' _filter = False _not = False alias = {'identifi': 'reg', 'register': 'reg', 'mask': 'match', 'ISP': 'isp', 'wikiblock': 'block', 'said': 'talk'} filters = alias.viewkeys() | alias.viewvalues() | {'ban', 'quiet', 'account', 'country', 'op', 'voic', 'affiliat', 'cloak', 'wmcloak'} i = 0 if words[i].lower() in ('list', 'which', 'who'): i += 1 word = words[i].rstrip('s').lower() if word in ('channel', 'account', 'cloak', 'ip', 'host', 'ident', 'nick', 'realname', 'connection', 'user'): command['type'] = word i += 1 if words[i] == 'here': if not here: return {'error': 'can not use "here" in pm'} command['channels'] = [here] i += 1 for count in range(100): word = words[i] if _filter: _filter -= 1 if word in ('not', 'non', 'without') or word.endswith('n\'t'): _not = True elif wordRoot(word) in filters: if word[0:2] == 'un': _not = True word = word in ('opped', 'voiced') and word or wordRoot(word) if word in alias: word = alias[word] command.setdefault('filters', []).append((_not and '!' or '') + word) _not, Filter = False, True if word in ('match', 'country', 'isp') and len(words) > i + 1: i += 1 command['filters'][-1] += ' ' + words[i] elif word == 'score' and len(words) > i + 2 and words[i + 1] in '=><' and words[i + 2].isdigit(): command.setdefault('filters', []).append((_not and '!' or '') + ' '.join(words[i:i + 3])) i += 2 elif word[0] in '+-' and (len(word) == 2 or prev in ('flag', 'flags')): command.setdefault('filters', []).append((_not and '!' or '') + 'flag ' + word + '') _not, Filter = False, False elif word == 'or' and 'filters' in command: command['filters'].append('or') elif ' '.join(words[i:i + 2]).lower() in ('in range', 'ip range', 'with ip') and len(words) > i + 2 \ and reIP.match(words[i + 2]) or word.lower() == 'ip' and len(words) > i + 1 and reIP.match(words[i + 1]): if ' '.join(words[i:i + 2]).lower() in ('in range', 'ip range', 'with ip'): ip = words[i + 2] i += 2 else: ip = words[i + 1] i += 1 command.setdefault('filters', []).append((_not and '!' or '') + 'ip ' + ip) elif word == 'in': channels = [] j = i + 1 _not = i > 0 and words[i - 1] == 'not' while j < len(words): chan = words[j].strip(',') if channels and chan == 'and': continue if chan[0] == '#': channels.append(chan) if chan in ('somewhere', 'anywhere') or (chan in ('any', 'some') and len(word) > j and word[j + 1] == 'channel'): channels.append('anychan') if chan in ('any', 'some'): j += 1 elif chan[0] == '-': autochans = sorted([(c, len(n)) for c, n in chanNicks.iteritems() if c.endswith(chan)], key=lambda i:i[1], reverse=True) if not autochans: return {'error': 'I don\'t know a channel that ends with ' + chan} channels.append(autochans[0][0]) else: i = j - 1 break j += 1 if 'filters' in command and command['filters'][-1].strip('!').startswith(('op', 'voiced', 'ban', 'quiet', 'flag +')): command['filters'][-1] += ' ' + ' '.join(channels) else: command[(_not and 'not ' or '') + 'channels'] = channels _not = False elif word in ('with', 'within', 'has', 'have') and len(words) > i + 2 and words[i + 1] == 'more': i += 2 word = words[i] if word == 'than' and len(words) > i + 2 and (words[i + 1].isdigit() or words[i + 1] in ('one', 'two')): word += ' %s %s' % ({'one': '1', 'two': '2'}.get(words[i + 1], words[i + 1]), words[i + 2]) i += 2 command['with more'] = word i += 1 prev = word if i == len(words): return command else: return {'error': 'Sorry, I didn\'t understand your command'} def wordRoot(word): if word[0:2] == 'un': word = word[2:] if word[-2:] == 'ed': if word[-3] == word[-4]: word = word[0:-3] else: word = word[0:-2] return word def queryBotDB(query): "Query data about users, channels, etc" if 'error' in query: return query['error'] if not query: return 'you need to provide channels and filters to query' if not query.viewkeys() - {'channels', 'not channels'}: return 'no filters found while parsing the command' channels = chanNicks.keys() if query.get('channels', ['anychan']) != ['anychan']: channels = [chan for chan in channels if chan in query['channels']] if 'not channels' in query: channels = [chan for chan in channels if not chan in query['not channels']] if not query.get('type', 'user') in ('user', 'account', 'cloak', 'ip', 'host', 'ident', 'nick', 'realname'): return 'Sorry, I didn\'t understand your query' users = [nickUser[nick] for nick in {nick for chan in channels for nick in chanNicks[chan]} if nick != 'ChanServ' and nick in nickUser] for test in query.get('filters', []): test, _not = test.strip('!'), test[0] == '!' if test.startswith('op ') or test == 'op': test = test.replace('op', 'flag +o', 1) word = test.split() chans = len(word) > 1 and (word[-1].startswith('#') or word[-1] == 'anychan') and [word[-1]] or channels if chans == ['anychan']: chans = chanNicks.keys() if test == 'reg': users = [u for u in users if not u.account] if _not else [u for u in users if u.account] elif test == 'talk': users = [u for u in users if not u.talked] if _not else [u for u in users if u.talked] elif test == 'cloak': users = [u for u in users if not u.get('cloak')] if _not else [u for u in users if u.get('cloak')] elif test == 'wmcloak': users = [u for u in users if not u.host.startswith(wikimediaCloaks)] if _not else \ [u for u in users if u.host.startswith(wikimediaCloaks)] elif test == 'affiliat': users = [u for u in users if u.get('cloak', 'unaffiliated') == 'unaffiliated'] if _not else \ [u for u in users if u.get('cloak', 'unaffiliated') != 'unaffiliated'] elif test.startswith('voiced'): c = test[7:] if c: voiced = {nick for nick, s in chanNicks[c].iteritems() if s > 0} users = [u for u in users if u.nick in chanNicks[c].viewkeys() - voiced] if _not else \ [u for u in users if u.nick in voiced] else: voiced = {nick for chan in chans for nick in chanNicks[chan] if chanNicks[chan][nick] > 0} users = [u for u in users if not u.nick in voiced] if _not else [u for u in users if u.nick in voiced] elif test.startswith('opped'): c = test[6:] if c: opped = {nick for nick, s in chanNicks[c].iteritems() if s > 1} users = [u for u in users if u.nick in chanNicks[c].viewkeys() - opped] if _not else \ [u for u in users if u.nick in opped] else: opped = {nick for chan in chans for nick in chanNicks[chan] if chanNicks[chan][nick] > 1} users = [u for u in users if not u.nick in opped] if _not else [u for u in users if u.nick in opped] elif test.startswith('ban '): bannd = {u.nick for u in users for chan in chans for ban, mask in chanBans.get(chan, []) if mask.match(u.full)} users = [u for u in users if not u.nick in bannd] if _not else [u for u in users if u.nick in bannd] elif test == 'blocks': blocks = [u for u in users if 'ip' in u and wikiBlock(u.ip)] users = [u for u in users if not u in blocks] if _not else blocks elif test.startswith('score '): testStr = test[6:] if testStr[0:2] == '= ': testStr = '=' + testStr temp = [u for u in users if eval('u.score ' + testStr + '')] users = [u for u in users if not u in temp] if _not else temp elif test.startswith('match '): mask = maskre(test[6:]) temp = [u for u in users if mask.match(u.full)] users = [u for u in users if not u in temp] if _not else temp elif test.startswith('flag'): testflags = word[1][1:] access = [('*' in entry and maskre(entry) or entry, args.get('flags', '')) for chan in chans for entry, args in chanList[chan]['access'].iteritems()] flagged = [] for u in users: for entry, flags in access: if (u.get('account', '').lower() == entry.lower() if type(entry) == str else entry.match(u.full)) and all(f in flags for f in testflags): flagged.append(u) break users = [u for u in users if u not in flagged] if _not else flagged elif test.startswith('account '): account = irclower(test[8:]) temp = [u for u in users if 'account' in u and irclower(u.account) == account] users = [u for u in users if not u in temp] if _not else temp elif test.startswith('ip '): ip = ip2numcdir(test[3:]) if not ip: self.msg('%s is not a valid IP' % test[3:]) return temp = [u for u in users if 'ip' in u and testip(u.ip, ip)] users = [u for u in users if not u in temp] if _not else temp elif test.startswith('country '): country = re.compile(re.escape(test[8:].strip('\'"')), re.I) temp = [u for u in users if 'country' in u and country.match(u.country)] users = [u for u in users if not u in temp] if _not else temp elif test.startswith('isp '): isp = re.compile(re.escape(test[4:].strip('\'"')), re.I) temp = [u for u in users if 'isp' in u and isp.match(u.isp)] users = [u for u in users if not u in temp] if _not else temp if not users: return 'there are no users that match these criteria' if query.get('type') in ('account', 'cloak', 'ip', 'host', 'ident', 'realname'): utype = {} for user in users: if query['type'] in user: utype.setdefault(user[query['type']], []).append(1) if query.get('with more') in ('nicks', 'connections'): ulist = sorted([(sum(l), t) for t, l in utype.items()], reverse=True) return mklist('%s (%d %s%s)' % (t, n, query['with more'][0:-1], n != 1 and 's' or '') for n, t in ulist[0:8]) utype = utype.keys() return mklist(len(utype) > 9 and utype[0:8] + ['%d more %ss' % (len(utype) - 8, query['type'])] or utype) else: return users def wherehasflag(user, flag): flag = set(flag) hasflag = set() if isinstance(user, dict): account = user.account user = user.full else: account = user for c in chanList: for entry in chanList[c]['access']: if (entry == account or '!' in entry and maskre(entry).search(user)) \ and flag <= set(chanList[c]['access'][entry]['flags']): hasflag.add(c) chanacs = [(c, i[9:]) for c in chanList for i in chanList[c]['access'] if i.startswith('$chanacs:') and flag <= set(chanList[c]['access'][i]['flags'])] for chan, pull in chanacs: if pull in chanList and account in chanList[pull]['access']: hasflag.add(chan) return sorted(hasflag, key=lambda c: len(chanNicks[c]), reverse=True) def whobanned(user, channel): if isinstance(user, dict): account = user.account user = user.full elif not '!' in user: user = findUser(user) if not user: return 'I don\'t know that user' account = user.account user = user.full else: account = None if not channel in chanList: return 'I don\'t know that channel' resp = [] for ban in chanList[channel]['ban']: match = False if ban[0] == '$': if ban[0:3] == '$a:' and account: match = True else: continue else: mask = maskre(ban) if mask.match(user): match = True if match: args = chanList[channel]['ban'][ban] date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(args['time']))) \ if args.get('time', '').isdigit() else '(unknown date)' comm = 'comment' in args and ' with comment "%s"' % args['comment'] or '' resp.append('%s was banned in %s by %s on %s%s' % (ban, channel, args['user'], date, comm)) return mklist(len(resp) > 3 and resp[0:2] + ['%d more bans found'] or resp) diff --git a/warns.py b/warns.py index abc909b..7d84ac7 100644 --- a/warns.py +++ b/warns.py @@ -1,455 +1,469 @@ # -*- coding: utf-8 -*- from database import * import tools reIdentIP = re.compile(r'!([0-9a-fA-F]{8})@.*/.*') +reASCII = re.compile(r'\|| ') def loadbl(): "Load the blacklist" global blacklist terms = [t[0] == '/' and t[-1] == '/' and t.strip('/') or '\\b' + re.escape(t) for t in chanList['global'].get('black', ())] try: blacklist = re.compile('|'.join(terms)) except: blacklist = None bot.msg('#wikimedia-opbot', 'Error while compiling blacklist') if not 'recentActions' in globals(): recentActions = defaultdict(lambda: deque(maxlen=30)) chanWarns = defaultdict(list) warnDelay = {} loadbl() chanDisabled = {} tracked = {} recentKRB = deque(maxlen=10) warned = deque(maxlen=20) warnedKRB = deque(maxlen=10) def mkping(channel, flag=None, inchan=['#wikimedia-opbot', '#wikimedia-ops'], noflag=None): "Return a set of channel ops nicks" if not channel in chanList or channel == '#wikimedia-overflow': return set() if isinstance(inchan, basestring): inchan = [inchan] accounts = {op for op in chanList[channel]['access'] if 'o' in chanList[channel]['access'][op]['flags'].lower()} - \ {'Sigyn', 'eir'} if any(1 for entry in accounts if entry.startswith('$chanacs:')): for entry in [entry for entry in accounts if entry.startswith('$chanacs:')]: chan = entry[9:] if chan in chanList: accounts |= {a for a in chanList[chan]['access'] if chanList[chan]['access'][a]['flags'] != 'b'} if flag and 'ping' in chanList['global']: flag = set(flag) accounts &= {a for a, args in chanList['global']['ping'].iteritems() if flag & set(args.get('flags', ''))} if noflag and 'ping' in chanList['global']: noflag = set(noflag) accounts -= {a for a, args in chanList['global']['ping'].iteritems() if noflag & set(args.get('flags', ''))} nicks = {nick for chan in inchan for nick in chanNicks.get(chan, ())} return {nick for nick in nicks if nickUser.get(nick, {}).get('account') in accounts} def warn(target, action, channel, ping=False, warnChan=None, delay=3): ping = ping and mkping(channel, ping) or set() for wnd in warned: if wnd[0:3] == (target, action, channel) and wnd[-1] + 30 > time.time(): print 'Warn Throttled: %s, %s, %s [%s]' % (target, action, channel, time.strftime('%H:%M:%S')) return warned.append((target, action, channel, time.time())) channel = type(channel) == list and channel or channel and [channel] or [] + if channel == ['#wikimedia-overflow']: + warnChan = None for chan in ['#wikimedia-opbot'] + (warnChan and (isinstance(warnChan, basestring) and [warnChan] or warnChan) or []): if chan in chanDisabled and ('all' in chanDisabled[chan] or not set(channel) - chanDisabled[chan]): print 'Warn disabled in %s: %s %s %s' % (chan, target, action, mklist(channel)) continue chanWarns[chan].append((target, action, channel, ping, time.time())) if delay: if not target in warnDelay: warnDelay[target] = reactor.callLater(delay, sendWarns, target) else: sendWarns(target) def sendWarns(target): if target in warnDelay: if warnDelay[target].active(): warnDelay[target].cancel() del warnDelay[target] now = time.time() # warn -> ('target', 'action', 'channel', set(ping), float(time)) # select all warns of the same target targetWarns = [(chan, w) for chan in chanWarns for w in chanWarns[chan] if w[0] == target and w[4] + 10 > now] if not targetWarns: return uniqueWarns = [] # merge warns to warn in the same channel for chan in {c for c, w in targetWarns}: cwarns = [w for c, w in targetWarns if c == chan] # if same target has warns with different actions in different channels, split the warn per action if len({w[1] for w in cwarns}) > 1 and len({repr(w[2]) for w in cwarns}) > 1: for action in {w[1] for w in cwarns}: uniqueWarns.append((chan, [w for w in cwarns if w[1] == action])) else: uniqueWarns.append((chan, cwarns)) chansUnique = [] for chan, warns in uniqueWarns: # if it is only one warn, add other warns with different target but same action and channel if len(warns) == 1: warns += [w for w in chanWarns[chan] if w[0] != target and w[1] == warns[0][1] and w[2] == warns[0][2] and w[4] + 10 > now] for w in warns: chanWarns[chan].remove(w) temp = [] nicks = mklist([w[0] for w in warns if not (w[0] in temp or temp.append(w[0]))]) actions = mklist([w[1] for w in warns if not (w[1] in temp or temp.append(w[1]))]) actions = re.sub(r' (?:in|from)(?=, | and)', '', actions) channels = mklist({c for w in warns for c in w[2]}) ping = set() if '\x0303' in actions else {n for w in warns for n in w[3]} # merge identic warns to send to diferent channels in one command (/PRIVMSG #chan1,#chan2 :message) # in order to avoid disconnection per Excess Flood for i, items in enumerate(chansUnique): if [nicks, actions, channels] == items[1:4]: chansUnique[i][0] += ',' + chan chansUnique[i][4] |= ping break else: chansUnique.append([chan, nicks, actions, channels, ping]) for chan, nicks, actions, channels, ping in chansUnique: ping = ' '.join(sorted(ping)) if 'IP blocked' in actions and nicks in nickUser and nickUser[nicks].country: actions = actions.replace('IP blocked', 'IP from %s blocked' % nickUser[nicks].country, 1) - if 'was ' in actions and ' ' in nicks: + if 'was ' in actions and ', ' in nicks or ' and ' in nicks: actions = re.sub(r'\bwas\b', 'were', actions) message = '%s %s %s%s' % (nicks, actions, channels, ping and '\x0315 ping %s\x03' % ping or '') if re.search('was \x0303(kick|removed|banned)', actions) and nicks in nickUser: inchans = [c for c in chanNicks if nicks in chanNicks[c]] if channels in inchans: message += ' and rejoined' elif inchans and not (nicks in nickUser and nickUser[nicks].account in chanList['#wikimedia-ops']['access']): message += ', they are also in ' + mklist(len(inchans) > 11 and inchans[0:9] + ['%d more channels' % (len(inchans) - 9)] or inchans) bot.msg(chan, message) def iswarned(target, action, channel, sec=60): now = time.time() for t, a, c, tm in warned: if (t, a, c) == (target, action, channel) and tm + sec > now: return True return False def join(user, channel): "Called when an user join a channel" identhost = '%(ident)s@%(host)s' % user ra = recentActions[identhost] ra.appendleft(('join', channel, time.time(), None)) towarn = [] ping = '' identIP = reIdentIP.search(user.full) if tools.isinlist(user, 'track'): entries = mklist(entry + ('comment' in args and ' "%s"' % args['comment'] or '') for entry, args in chanList['global']['track'].iteritems() if (user.account == entry[3:] if entry[0:3] == '$a:' else '!' in entry and maskre(entry).match(user.full))) towarn.append('tracked (%s)' % entries) ping += 't' elif user.host.startswith(wikimediaCloaks) or tools.isinlist(user, 'exempt'): return elif identhost in tracked and tracked[identhost][1] + 3600 > time.time(): towarn.append('recently ' + tracked[identhost][0]) ping += 't' elif ('ip' in user or identIP) and user.score < 10: ip = user.ip or '%d.%d.%d.%d' % tuple(int(identIP.group(1)[i:i + 2], 16) for i in (0, 2, 4, 6)) blocks = tools.wikiBlock(ip) if blocks: towarn.append('IP blocked in ' + blocks) ping += 'w' wikis = ' '.join(set(re.findall('global|\w+wiki', blocks))) tools.logAction(user=user, channel=channel, action='ip-block', target=wikis) target = user.full globalBans = [ban for ban, mask in tools.chanBans['#wikimedia-bans'] if mask.match(target)] if user.account: accban = '$a:' + irclower(user.account) globalBans += [ban for ban in chanList['#wikimedia-bans']['ban'] if ban[0] == '$' and irclower(ban) == accban] if globalBans: towarn.append('global banned (%s)' % ', '.join(globalBans)) ping += 'g' ra.appendleft(('gbanned join', channel, time.time(), None)) + nickdigit = re.search(r'(\D+)\d+', user.nick) + if (nickdigit and user.ident == '~' + nickdigit.group(1) or user.ident == ('~' + user.nick[0:-1])[0:10] + and user.nick[-1].isdigit()) and not '/' in user.host: + towarn.append('possible spambot') action = '(%s) \x0307joined\x03' % mklist(towarn) if towarn and ('tracked' in towarn or not iswarned(user.nick, action, channel, 86400)): warn(user.nick, action, channel, ping) def part(user, channel, reason=''): "Called when an user leaves a channel" identhost = '%(ident)s@%(host)s' % user ra = recentActions[identhost] match = re.match(r'requested by (\S+)', reason) if match: ra.appendleft(('removed', channel, time.time(), user.nick)) rt = time.time() recentKRB.appendleft(('removed', user.full, channel, match.group(1), rt)) tools.logAction(user=nickUser[match.group(1)], channel=channel, action='remove', target=user) for act, banned, chan, by, tm in recentKRB: if not (act == 'banned' and tm + 15 > time.time() and (channel == chan or channel in jBans.get(chan, ())) and maskre(banned).match(user.full)) or tm in warnedKRB: continue warnedKRB.append(rt) warnedKRB.append(tm) if by == match.group(1): warn(user.nick, 'was \x0303banned and removed\x03 by %s from' % match.group(1), channel, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) else: warn(user.nick, 'was \x0303banned\x03 by %s and \x0303removed\x03 by %s from' % (by, match.group(1)), channel, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) kickbanned(user, channel) break else: def removeWarn(): for act, chan, tm, nick in ra: if act == 'kickban' and tm + 15 > time.time(): break else: warn(user.nick, 'was \x0303removed\x03 by %s from' % match.group(1), channel, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=0) warnedKRB.append(rt) reactor.callLater(15, removeWarn) return ra.appendleft(('part', channel, time.time(), user.nick)) if tools.isinlist(user, 'track'): warn(user.nick, '\x0303parted\x03', channel, delay=1) return timeCheck = time.time() - 5 last5sec = [act for act, chan, tm, arg in ra if chan == channel and tm > timeCheck] if 'join' in last5sec and 'msg' in last5sec: warn(user.nick, '\x0307spammed (join-say-part)\x03', channel, 's', warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') elif user.full == 'Sigyn!sigyn@freenode/utility-bot/sigyn': bot.msg('#wikimedia-ops', 'Sigyn parted from ' + channel) def quit(user, channels, reason): "Called when an user quit" identhost = '%(ident)s@%(host)s' % user recentActions[identhost].appendleft(('quit', reason, time.time(), user.nick)) if reason.startswith('Killed ('): by = reason.split('(')[1].strip() if by.endswith('.freenode.net') or reason == 'Killed (Sigyn (BANG!))': return - warn(user.nick, 'was \x0303killed\x03 by %s in' % by, channels, warnChan='#wikimedia-ops', delay=1) + warn(user.nick, 'was \x0303killed\x03 by %s in' % by, channels, delay=1) elif reason == 'K-Lined': - warn(user.nick, 'was \x0303K-Lined\x03 in', channels, warnChan='#wikimedia-ops', delay=1) + warn(user.nick, 'was \x0303K-Lined\x03 in', channels, delay=1) elif tools.isinlist(user, 'track'): warn(user.nick, '\x0303quit\x03 (%s) from' % reason, channels, delay=1) def msg(user, channel, msg): "Called when an user send a message to a channel" # ignore users with affiliated cloak if user.get('cloak', 'unaffiliated') != 'unaffiliated' or tools.isinlist(user, 'exempt') \ or chanNicks[channel].get(user.nick): return now = time.time() + ra = recentActions['%(ident)s@%(host)s' % user] + ra.appendleft(('msg', channel, now, msg)) + if msg.count('\x03') > 5: + warn(user.nick, '\x0307color spam\x03', channel, 's', + warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') + if chanNicks[channel]['wmopbot'] >= 2: + autokick(user, channel, 'color spam') + return + if len(reASCII.findall(msg)) > 5 and len(ra) > 1 and (ra[1][0], ra[1][1]) == ('msg', channel) and \ + len(reASCII.findall(ra[1][3])) > 5 and len(msg) / (len(re.findall(r'\w', msg)) + 1) > 2: + warn(user.nick, '\x0307ASCII art\x03 in', channel, 's', + warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') + if chanNicks[channel]['wmopbot'] >= 2: + autokick(user, channel, 'ASCII art spam') + return + if (msg.count('\xc3') + msg.count('\xc2')) * 3 > len(msg) > 20: + warn(user.nick, '\x0307unicode spam\x03', channel, 's', + warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') + return + if msg.split(' ')[0].endswith(':'): + nickping = msg.split(' ')[0][0:-1] + if nickping and not nickping.strip('@') in chanNicks[channel]: + warn(user.nick, '\x0307pinging user not in channel\x03', channel, 's') + return + if len(ra) > 2 and ra[1][1] == channel and ra[1][0] in ('msg', 'join') and len(msg) / (now - ra[1][2]) > 8: + warn(user.nick, '\x0307too fast typing\x03', channel) nohttp = re.sub(r'https?://\S+', '', msg) repChar = re.search(r'(\S)\1{8,}', nohttp) tooConsonants = re.search(r'[bcdfghjklmnpqrstvwxz]{8,}', nohttp) blmatch = blacklist and blacklist.search(msg) m = blmatch or repChar or tooConsonants if m: action = blmatch and 'said a \x0307blacklisted term\x03' or repChar and '\x0307repeated character\x03' \ or tooConsonants and '\x0307many consective consonants\x03' start = re.search(r'\S+ ?$', msg[0:m.start()]) end = re.search(r'^ ?\S+', msg[m.end():]) match = (start and start.group(0) or '') + '\x1f' + m.group(0)[0:100] + '\x1f' + (end and end.group(0) or '') if user.get('score', 0) < 2: warn(user.nick, '%s (%s\x0f) in' % (action, match), channel, 'b', - warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) + warnChan=not channel.startswith(('#wikimedia-opbot', '#wikimedia-overflow')) + and channel != '#wikimedia-ops' and '#wikimedia-ops', delay=1) tools.logAction(user=user, channel=channel, action='blacklist', target=m.group(0), note=msg) else: warn('%s (score %d)' % (user.nick, user.get('score', 0)), '%s (%s\x0f) in' % (action, match[0:100]), channel, False, delay=1) return - ra = recentActions['%(ident)s@%(host)s' % user] - ra.appendleft(('msg', channel, now, msg)) if len([(a, t) for a, c, t, m in ra if c == channel and a == 'msg' and t + 20 > now]) > 10 and \ not iswarned(user.nick, '\x0307flooding\x03', channel): warn(user.nick, '\x0307flooding\x03', channel, 's', warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') return if len(ra) > 2 and (ra[1][0], ra[1][1], ra[1][3]) == ('msg', channel, msg) == (ra[2][0], ra[2][1], ra[2][3]): warn(user.nick, '\x0307repeating message\x03 "%s" in' % (len(msg) > 50 and msg[0:47] + '...' or msg), channel, 's', warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') return - if msg.count('\x03') > 5: - warn(user.nick, '\x0307color spam\x03', channel, 's', - warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') - if chanNicks[channel]['wmopbot'] >= 2: - autokick(user, channel, 'color spam') - return - if msg.count('|') + msg.count(' ') > 5 and len(ra) > 1 and (ra[1][0], ra[1][1]) == ('msg', channel) and \ - ra[1][3].count('|') + ra[1][3].count(' ') > 5 and not re.search(r'[a-z]', msg + ra[1][3]): - warn(user.nick, '\x0307ASCII art\x03 in', channel, 's', - warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') - if chanNicks[channel]['wmopbot'] >= 2: - autokick(user, channel, 'ASCII art spam') - return pings = len(set(msg.split()) & chanNicks[channel].viewkeys()) if pings > 5 or pings > 2 and any(1 for a, c, t, m in ra if c == channel and a == 'ping3' and t + 20 > now): warn(user.nick, '\x0307ping spam\x03', channel, 's', warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') if chanNicks[channel]['wmopbot'] >= 2: autokick(user, channel, 'ping spam') return if pings > 2: ra.appendleft(('ping3', channel, now, None)) def renamed(user, oldNick): "Called when an user nick is changed" recentActions['%(ident)s@%(host)s' % user].appendleft(('renamed', None, time.time(), oldNick)) def notice(user, channel, msg): "Called when an user send a notice to a channel" if user.get('cloak', 'unaffiliated') != 'unaffiliated' or tools.isinlist(user, 'exempt'): return recentActions['%(ident)s@%(host)s' % user].appendleft(('notice', None, time.time(), msg)) if user.account in ('wmopbot', 'ChanServ', 'Sigyn', 'eir'): return if not iswarned(user.nick, '\x0307sent notice\x03 to', channel): if channel.startswith('#wikimedia-opbot') or 'AntiSpamMeta' in chanNicks[channel]: warn(user.nick, '\x0307sent notice\x03', channel, 's') else: warn(user.nick, '\x0307sent notice\x03', channel, 's', warnChan='#wikimedia-ops') def banned(user, channel, target): "Called when I see a ban action" if target == '*!*@*': return if '$' in target[1:]: - target = target[0:target.find('$')] + target = target[0:target.find('$', 1)] bt = time.time() recentKRB.appendleft(('banned', target, channel, user.nick, bt)) if target.startswith('$a:'): account = target[3:] banUsers = [u for u in nickUser.itervalues() if u.account == account] elif target[0] != '$' and '!' in target: mask = maskre(target) if not mask: return banUsers = [u for u in nickUser.itervalues() if mask.match(u.full)] else: return banChans = [channel] + jBans.get(channel, []) for target in banUsers: ra = recentActions['%(ident)s@%(host)s' % target] ra.appendleft(('banned', channel, time.time(), target)) for act, kicked, chan, by, tm in recentKRB: if not (act in ('kicked', 'removed') and chan in banChans and target.full.endswith(kicked.split('!')[1]) and tm + 15 > time.time()) or tm in warnedKRB: continue warnedKRB.append(tm) warnedKRB.append(bt) + tgt = target.nick + (target.ip and ' (%s)' % target.ip or '') if act == 'kicked' and by == user.nick: - warn(target.nick, 'was \x0303kickbanned\x03 by %s from' % user.nick, chan, + warn(tgt, 'was \x0303kickbanned\x03 by %s from' % user.nick, chan, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) elif act == 'removed' and by == user.nick: - warn(target.nick, 'was \x0303removed and banned\x03 by %s from' % user.nick, chan, + warn(tgt, 'was \x0303removed and banned\x03 by %s from' % user.nick, chan, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) else: - warn(target.nick, 'was \x0303%s\x03 by %s and \x0303banned\x03 by %s from' % (act, by, user.nick), + warn(tgt, 'was \x0303%s\x03 by %s and \x0303banned\x03 by %s from' % (act, by, user.nick), chan, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) kickbanned(target, channel) def kicked(user, channel, target, reason): "Called when I see a kick action" ra = recentActions['%(ident)s@%(host)s' % target] ra.appendleft(('kicked', channel, time.time(), target.nick)) kt = time.time() recentKRB.appendleft(('kicked', target.full, channel, user.nick, kt)) + tgt = target.nick + (target.ip and ' (%s)' % target.ip or '') for act, banned, chan, by, tm in recentKRB: if not (act == 'banned' and tm + 15 > time.time() and (channel == chan or channel in jBans.get(chan, ())) and maskre(banned).match(target.full)) or tm in warnedKRB: continue if by == user.nick: - warn(target.nick, 'was \x0303kickbanned\x03 by %s from' % user.nick, channel, + warn(tgt, 'was \x0303kickbanned\x03 by %s from' % user.nick, channel, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) else: - warn(target.nick, 'was \x0303banned\x03 by %s and \x0303kicked\x03 by %s from' % (by, user.nick), channel, + warn(tgt, 'was \x0303banned\x03 by %s and \x0303kicked\x03 by %s from' % (by, user.nick), channel, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=1) warnedKRB.append(kt) warnedKRB.append(tm) kickbanned(target, channel) break else: def kickWarn(): for act, chan, tm, nick in ra: if act == 'kickban' and tm + 15 > time.time(): break else: - warn(target.nick, 'was \x0303kicked\x03 by %s from' % user.nick, channel, + warn(tgt, 'was \x0303kicked\x03 by %s from' % user.nick, channel, warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops', delay=0) tracked['%(ident)s@%(host)s' % target] = ('kicked from ' + channel, time.time()) warnedKRB.append(kt) reactor.callLater(15, kickWarn) def kickbanned(target, channel): "Called after kickban warning" identhost = '%(ident)s@%(host)s' % target ra = recentActions[identhost] ra.appendleft(('kickban', channel, time.time(), target.nick)) tracked[identhost] = ('kickbanned from ' + channel, time.time()) ftarget = target.full globalBans = [ban for ban, mask in tools.chanBans['#wikimedia-bans'] if mask.match(ftarget)] sugBan = 'ip' in target and ['*!*@' + (target.host.startswith('gateway/') and '*' or '') + target.ip] or [] if target.account: accban = '$a:' + target.account globalBans += [ban for ban in chanList['#wikimedia-bans']['ban'] if ban == accban] sugBan += [accban] sugBan = [b for b in sugBan if not b in globalBans] if len({chan for act, chan, tm, arg in ra if act == 'kickban' and not chan.startswith('#wikimedia-opbot')}) > 1 \ and sugBan and not iswarned(identhost, 'suggest gban', '#wikimedia-ops'): message = 'Two recent kickbans for %s, I suggest to global ban %s \x0315ping %s' % (target.nick, mklist(sugBan), mklist(sorted(mkping('#wikimedia-bans', False, inchan='#wikimedia-ops')))) warned.append((identhost, 'suggest gban', '#wikimedia-ops', time.time())) print '[%s] WARN: #wikimedia-ops %s' % (time.strftime('%Y-%m-%d %H:%M:%S'), message) reactor.callLater(2, bot.msg, '#wikimedia-ops', message) def topic(user, channel, new): if user.host == 'services.': return if user.get('cloak', 'unaffiliated') != 'unaffiliated' or tools.isinlist(user, 'exempt') \ or chanNicks[channel].get(user.nick): return if not 't' in chanList[channel]['config']['cs/irc'].get('modes', '') and not 'cloak' in user \ and not tools.isinlist(user, 'exempt'): warn(user.nick, '(%s) \x0307changed the topic\x03' % ('unreg' in user and 'unregistered' or 'unaffiliated'), channel, 's', warnChan=not channel.startswith('#wikimedia-opbot') and '#wikimedia-ops') def autokick(target, channel, reason='', ban=False): "Process kickban after first checks and whois query processed" print 'warns kickban', target.full, channel, reason if channel == '#wikimedia-overflow': print 'Skiping autokick in -overflow' return if not channel in chanList or not 'o' in chanList[channel]['access'].get('wmopbot', {}).get('flags', '').lower() \ and chanNicks[channel]['wmopbot'] < 2: print 'Error: autokick where wmopbot is not op' return if target.score > 100: print 'Error: autokick on high scored user', target return if ban: ban = 'account' in target and '$a:' + target.account or 'ip' in target and \ (target.host.startswith('gateway/') and '*!*@*' or '*!*@') + target['ip'] or '*!*@' + target.host if ban == '*!*@*': # for safety print '*!*@* ban error' ban = False - def oped(): - if ban: - bot.sendLine('MODE %s +b %s' % (channel, ban)) - bot.sendLine('KICK %s %s %s' % (channel, target.nick, reason and ':' + reason)) - print 'applying auto kick' + (ban and 'ban' or '') + if ban: + bot.sendLine('MODE %s +b %s' % (channel, ban)) + bot.sendLine('KICK %s %s %s' % (channel, target.nick, reason and ':' + reason)) + print 'applying auto kick' + (ban and 'ban' or '') if 'eir' in chanNicks[channel]: try: tools.eirComment[('ban', ban)] = 'wmopbot autokick' + (reason and ' (%s)' % reason) except: print 'Error on set eir comment' - if [1 for nick in bot.nicks if (nick, 2) in chanNicks[channel].items()]: - oped() - else: - addhook('op ' + channel, oped) - bot.sendLine('CS OP ' + channel) - reactor.callLater(3, bot.sendLine, 'CS DEOP ' + channel) diff --git a/web.py b/web.py index 02c5246..e67b435 100755 --- a/web.py +++ b/web.py @@ -1,595 +1,618 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from flask import Flask, render_template, session, redirect, request, url_for, request from collections import defaultdict from requests_oauthlib import OAuth1 -import time, re, oursql, os, requests, jwt, thread +import time, re, oursql, os, requests, jwt, thread, sys def _force_https(app): def wrapper(environ, start_response): environ['wsgi.url_scheme'] = 'https' return app(environ, start_response) return wrapper app = Flask(__name__) app.wsgi_app = _force_https(app.wsgi_app) app.debug = False app.config['PREFERRED_URL_SCHEME'] = 'https' _dbconn = [0, None] chanLoaded = 0 sep = re.compile(r'^;|(?<=[^\\]);') if 'userCheck' not in globals(): userCheck = {} threadQueue = [] lock = thread.allocate_lock() userType = {} @app.route('/login', methods=['GET', 'POST']) def login(): error = None with open('web.log', 'a') as f: f.write('login: %s; %r\n' % (request.method, session.items())) # make login if request.method == 'POST': session.permanent = request.form.get('permanent') == 'yes' returnto = request.form.get('returnto', '').lstrip('/') uri = url_for('index') + returnto if not 'user' in session: with open('login_tokens') as f: for token, user, t in [line.split('\t') for line in f.read().split('\n') if line]: if token == session.get('token'): if not user: error = 'Login failed' break session['user'] = user del session['token'] return redirect(uri) elif not error: return redirect(uri) # show login command returnto = request.args.get('returnto') try: with open('login_tokens') as f: tokens = [i for i in [line.split('\t') for line in f.read().split('\n') if line] if int(i[2]) > time.time()] except IOError: tokens = [] token = os.urandom(5).encode('hex') session['token'] = token tokens.append([token, '', '%d' % (time.time() + 60)]) with open('login_tokens', 'w') as f: f.write('\n'.join('\t'.join(i) for i in tokens)) return render_template('login.html', token=token, error=error, returnto=returnto) @app.route('/logout') def logout(): if 'user' in session: del session['user'] return render_template('logout.html') @app.route('/') def index(): if not 'user' in session: return redirect_login() with open('.online') as f: online = f.read().split('\n') if not online[0].isdigit() or int(online[0]) + 700 < time.time(): online = False r = query("""SELECT * FROM actions WHERE ac_timestamp > DATE_SUB(NOW(), INTERVAL 1 DAY) AND ac_action IN ('ban', 'kick', 'Killed', 'K-Lined', 'revove') AND ac_channel NOT LIKE '#wikimedia-opbot%' ORDER BY ac_id DESC LIMIT 20""")[::-1] graph = query("SELECT st_hour, SUM(st_msg) FROM stats WHERE st_hour > ? GROUP BY st_hour", ((int(time.time()) / 3600) - 168,)) if graph: graph = '{%s}' % ','.join('%d:%d' % (h, n) for h, n in graph) queuelen = None if chanList['#wikimedia-ops']['access'].get(session['user'], {}).get('template', '') == 'Contact' \ or query("SELECT 1 FROM user WHERE us_account = ? and us_user LIKE '%@freenode/staff/%' LIMIT 1", (session['user'],)): queuelen = query("SELECT COUNT(*) FROM cloak WHERE cl_status = 'new'")[0] return render_template('mainpage.html', title='wmopbot Web Interface', actions=r, graph=graph, online=online and online[1:], **userparams()) @app.route('/channels') def channels(): if not 'user' in session: return redirect_login() if chanLoaded + 300 < time.time(): loadChans() return render_template('channels.html', title=u'Wikimedia IRC channels', chanTable=chanTable, **userparams()) @app.route('/actions') def actions(): if not 'user' in session: return redirect_login() # actions: id, user(90), channel(50), action(10), target(90), args(255), timestamp where, args, search, limit = [], [], [], 50 for column in ('user', 'action', 'channel', 'target', 'args', 'timestamp'): value = request.args.get(column) if not value: continue search.append('%s: %s' % (column, value)) if '*' in value or '?' in value: value = value.replace('%', r'\%').replace('_', r'\_').replace('*', '%').replace('?', '_') where.append('ac_%s LIKE ?' % column) args.append(value) else: where.append('ac_%s = ?' % column) args.append(value) if 'limit' in request.args and request.args['limit'].isdigit(): limit = int(request.args['limit']) if limit > 2000: limit = 2000 r = query('SELECT * FROM actions%s ORDER BY ac_id DESC LIMIT %d' % (where and ' WHERE ' + ' and '.join(where) or '', limit), tuple(args))[::-1] return render_template('actions.html', title='Actions log', actions=r, search=search, **userparams()) @app.route('/lists') def botlists(): if not 'user' in session: return redirect_login() r = query("SELECT ls_type, ls_target, ls_args FROM list WHERE ls_channel = 'global' and ls_type IN ('black', 'bot', 'exempt', 'track')") lists = [(_type, target, {k: v.decode('utf-8', 'replace').replace('\;', ';') for k, v in (i.split(':', 1) for i in sep.split(args))}) for _type, target, args in r] lists = {_type: [(t.decode('utf-8', 'replace'), a.get('user'), a.get('time')) for tp, t, a in lists if tp == _type] for _type in {a for a, b, c in lists}} return render_template('lists.html', title='Lists', lists=lists, **userparams()) @app.route('/info') @app.route('/info/') def info(target=None): if not 'user' in session: return redirect_login() if not target: return render_template('info.html', title='Info', **userparams()) if target.startswith('#'): return channel(target) elif not any(x in target for x in '!@*?/.:'): r = query('SELECT * FROM user WHERE LOWER(us_account) = ? LIMIT 100', (target.lower(),)) access = [(c, chanList[c]['access'][target]) for c in chanList if target in chanList[c]['access']] accban = '$a:' + target bans = [(chan, ban, chanList[chan]['ban'][ban]) for chan in chanList for ban in chanList[chan]['ban'] if ban == accban] quiets = [(chan, quiet, chanList[chan]['quiet'][quiet]) for chan in chanList for quiet in chanList[chan]['quiet'] if quiet == accban] if r or access or bans: return render_template('user.html', title='Info about ' + target, users=r, access=access, bans=bans, quiets=quiets, **userparams()) else: return render_template('info.html', title='Info about ' + target, users=r, warn=target + ' is an unknown account', **userparams()) elif '!' in target or '@' in target: mask = target.replace('%', r'\%').replace('_', r'\_').replace('*', '%').replace('?', '_') r = query('SELECT * FROM user WHERE us_user LIKE ? LIMIT 100', (mask,)) actions = query('SELECT * FROM actions WHERE ac_user LIKE ? OR ac_target LIKE ? ORDER BY ac_id DESC LIMIT 20', (mask, mask))[::-1] bans = [(chan, ban, chanList[chan]['ban'][ban]) for chan in chanList for ban in chanList[chan]['ban'] if maskre(ban, target)][:100] quiets = [(chan, quiet, chanList[chan]['quiet'][quiet]) for chan in chanList for quiet in chanList[chan]['quiet'] if maskre(quiet, target)][:100] return render_template('mask.html', title='Info about ' + target, users=r, bans=bans, quiets=quiets, actions=actions, **userparams()) else: return render_template('info.html', title='Info', **userparams()) def channel(chan): if chan in chanList: del chanList[chan] for chan, _type, target, args in query('SELECT * FROM list WHERE ls_channel = ?', (chan,)): chanList[chan].setdefault(_type, {})[target.decode('utf-8', 'replace')] = \ {k: v.decode('utf-8', 'replace').replace('\;', ';') for k, v in (i.split(':', 1) for i in sep.split(args))} if chan in chanList: actions = query("SELECT * FROM actions WHERE ac_channel = ? AND ac_action IN ('ban', 'quiet', 'Killed', 'K-Lined', 'kick', 'remove') ORDER BY ac_id DESC LIMIT 10", (chan,))[::-1] graph = query("SELECT st_hour, st_msg FROM stats WHERE st_hour > ? AND st_channel = ?", ((int(time.time()) / 3600) - 168, chan)) if graph: graph = '{%s}' % ','.join('%d:%d' % (h, n) for h, n in graph) return render_template('channel.html', title='%s info' % chan, chan=chanList[chan], actions=actions, channel=chan, graph=graph, **userparams()) else: msg = chan + ' is an unknown channel, maybe the channels is not in meta:IRC/Channels.' return render_template('info.html', title='Info', warn=msg, **userparams()) @app.route('/lang') def languages(): if not 'user' in session: return redirect_login() messages = [key.split('.', 1) + [msg, user, dt.strftime('%Y-%m-%d %H:%M:%S')] for key, msg, user, dt in query('SELECT * FROM lang')] return render_template('lang.html', title='Languages messages', messages=messages, **userparams()) @app.route('/help') def help(): if not 'user' in session: return redirect_login() return render_template('help.html', title='wmopbot help', **userparams()) @app.route('/about') def about(): return render_template('about.html', title='What is wmopbot') @app.route('/cloak', methods=['GET', 'POST']) def cloakrequest(): if request.method == 'POST': if not 'cloak' in request.form: return render_template('cloak.html', message='To request a cloak, use the follow command in IRC: /msg wmopbot cloak') token = request.form['token'] with open('cloak.ctrl') as f: ctrl = f.readlines() for i, line in enumerate(ctrl): line = line.rstrip('\n').split('\t') if line[0] == token: if line[-1].isdigit(): return render_template('cloak.html', message='Error: This request had been already sent.') ircuser, ircacc, wikiacc, botacc = line[3], line[4], line[5], len(line) > 6 and line[6] or None + ctrl[i] = '\t'.join(line + ['%d' % time.time()]) + '\n' break else: return render_template('cloak.html', message='Internal Error') # token not in cloak.ctrl data = '%s: request %s\n' % (ircacc, time.strftime('%Y-%m-%d %H:%M:%S')) if wikiacc in userCheck and userCheck[wikiacc][1]: data += userCheck[wikiacc][1] # cloak: cl_id (key), cl_user (90), cl_ircacc (16), cl_wikiacc (tinytext), cl_ircbot (16), cl_type(enum), # cl_cloak (50), cl_status (10), cl_timestamp (datetime), cl_data (text) sql = 'INSERT INTO cloak (cl_user, cl_ircacc, cl_wikiacc, cl_ircbot, cl_type, cl_cloak, cl_status, cl_data) VALUES (?, ?, ?, ?, ?, ?, "new", ?)' - execute(sql, (ircuser, ircacc, wikiacc, botacc, request.form['type'], request.form['cloak'], data)) + try: + args = tuple((type(i) == unicode and i.encode('utf-8') or i) for i in (ircuser, ircacc, wikiacc, botacc, + request.form['type'], request.form['cloak'], data)) + execute(sql, args) + except UnicodeEncodeError: + sys.stderr.write(repr((ircuser, ircacc, wikiacc, botacc, request.form['type'], request.form['cloak'], data)) + '\n') + execute(sql, (ircuser, ircacc, wikiacc, botacc, request.form['type'], request.form['cloak'], data)) return render_template('cloak.html', message='The request was sent. The cloaks are processed in batches, so it may take some days to your request be processed.') # method == GET if 'ircacc' in request.args and 'wikiacc' in request.args: # for tests ircuser, ircacc, wikiacc = request.args.get('ircuser', '?!?@?'), request.args['ircacc'], request.args['wikiacc'] token, cloak = os.urandom(5).encode('hex'), None with open('cloak.ctrl', 'a') as f: f.write('%s\t?\t%d\t%s\t%s\t%s\n' % (token, time.time(), ircuser, ircacc, wikiacc)) elif not 'oauth_token' in request.args or not 'oauth_verifier' in request.args: return render_template('cloak.html', message='To request a cloak, use the follow command in IRC: /msg wmopbot cloak') else: token = request.args['oauth_token'] with open('cloak.ctrl') as f: ctrl = [l for l in f.readlines() if l and int(l.split()[2]) + 25920000 > time.time()] for line in ctrl: line = line.rstrip('\n').split('\t') if line[0] == token: if len(line) > 5: return render_template('cloak.html', message='Error: Request already initialized.') if int(line[2]) + 120 < time.time(): return render_template('cloak.html', message='Error: Request expired.') token_secret, ircuser, ircacc = line[1], line[3], line[4] break else: return render_template('cloak.html', message='Error: Not a valid token') with open('.oauth.key') as f: consumer_key, consumer_secret = f.read().rstrip('\n').split('\t') oauth = OAuth1(consumer_key, consumer_secret, token, token_secret, verifier=request.args['oauth_verifier']) url = 'https://meta.wikimedia.org/w/index.php' r = requests.post(url=url, params={'title': 'Special:OAuth/token'}, auth=oauth) t = r.content.startswith('oauth_token') and dict(i.split('=', 1) for i in r.content.split('&')) oauth = OAuth1(consumer_key, consumer_secret, t['oauth_token'], t['oauth_token_secret']) r = requests.post(url=url, params={'title': 'Special:OAuth/identify'}, auth=oauth) data = jwt.decode(r.content, consumer_secret, audience=consumer_key) wikiacc = data['username'] for i, line in enumerate(ctrl): line = line.rstrip('\n').split('\t') if line[0] == token: - ctrl[i] = '\t'.join(line + [wikiacc]) + '\n' + ctrl[i] = '\t'.join(line + [wikiacc.encode('utf-8')]) + '\n' with open('cloak.ctrl', 'w') as f: - f.writelines(ctrl) + try: f.writelines(ctrl) + except Exception as e: + sys.stderr.write(repr(ctrl)) + raise e wikimediaCloaks = ('wikipedia/', 'wikimedia/', 'wikibooks/', 'wikinews/', 'wikiquote/', 'wiktionary/', - 'wikisource/', 'wikivoyage/', 'wikiversity/', 'wikidata/', 'mediawiki/') + 'wikisource/', 'wikivoyage/', 'wikiversity/', 'wikidata/', 'mediawiki/', 'wikispecies/') host = ircuser.split('@')[-1] cloak = host.startswith(wikimediaCloaks) and host or '' checkuser(wikiacc) return render_template('cloak.html', ircuser=ircuser, ircacc=ircacc, wikiacc=wikiacc, token=token, usercloak=cloak) @app.route('/botaccount', methods=['POST']) def botaccount(): token = request.form['token'] with open('cloak.ctrl') as f: ctrl = [l for l in f.readlines() if l and int(l.split()[2]) + 25920000 > time.time()] for line in ctrl: line = line.rstrip('\n').split('\t') if line[0] == token: if len(line) < 7: return 'Error: bot identification not received' else: return line[6] return 'Error: token not found' @app.route('/cloakqueue') def cloakqueue(): if not 'user' in session: return redirect_login() uparams = userparams() usergroup = uparams['userType'] if not usergroup in ('gc', 'staff', 'dev'): return render_template('base.html', title='Cloaks Queue', content='

The cloak queue is restricted to GCs and Freenode staff

', **uparams) limit = 'limit' in request.args and request.args['limit'].isdigit() and int(request.args['limit']) or 20 args = [limit > 200 and 200 or limit] status = request.args.get('status', usergroup == 'staff' and 'approved' or 'new') if status != 'all': args[0:0] = [status] queue = query('SELECT * FROM cloak WHERE cl_status = ? ORDER BY cl_id DESC LIMIT ?', tuple(args)) else: queue = query('SELECT * FROM cloak ORDER BY cl_id DESC LIMIT ?', tuple(args)) + queue = [tuple((type(i) == str and i.decode('utf-8') or i) for i in line) for line in queue] for req in queue: data = req[-1].split('\n') if len(data) < 2 or not data[1]: checkuser(req[3]) return render_template('cloakqueue.html', title='Cloaks Queue', limit=limit, queuestatus=status, requests=queue[::-1], usergroup=usergroup, **uparams) @app.route('/setcloak', methods=['POST']) def setcloak(): if not 'user' in session: return 'Error: not logged' with open('web.log', 'a') as f: f.write('setcloak: %r\n' % request.form) _id = request.form['id'] status, cloak = request.form.get('status'), request.form.get('cloak') r = query('SELECT cl_data FROM cloak WHERE cl_id = ?', (_id,)) if not r: return 'Error: id not found' data = r[0][0].split('\n', 1) history = data[0].split(',') new = '' if chanList['#wikimedia-ops']['access'].get(session['user'], {}).get('template', '') == 'Contact': if cloak: new = '%s: cloak edit %s' % (session['user'], time.strftime('%Y-%m-%d %H:%M:%S')) if history and history[-1].startswith('%s: cloak edit ' % session['user']): history = history[0:-1] data = ','.join(history + [new]) + '\n' + data[1] execute('UPDATE cloak SET cl_cloak_edit = ?, cl_data = ? WHERE cl_id = ?', (cloak, data, _id)) return '(ok)%s\n%s' % (cloak, new) elif not status in ('approved', 'rejected', 'cloaked'): return 'Error: You can not set that status' elif not status: return 'Error: No status to set' elif query("SELECT 1 FROM user WHERE us_account = ? and us_user LIKE '%@freenode/staff/%'", (session['user'],)): if status != 'cloaked': return 'Error: can not set that status' else: return 'Error: You can not set that status' new = '%s: %s %s' % (session['user'], status, time.strftime('%Y-%m-%d %H:%M:%S')) - data = ','.join(history + [new]) + '\n' + data[1] + data = ','.join(history + [new.encode('utf-8')]) + '\n' + data[1] status = status in ('+1', '-1') and 'reviewed' or status execute("UPDATE cloak SET cl_status = ?, cl_data = ? WHERE cl_id = ?", (status, data, _id)) return '(ok)' + new @app.route('/redundant') def redundantBans(): "Generate a list of redundant bans for all channels" if not 'user' in session: return redirect_login() bans, quiets = [], [] for channel in sorted(c for c in chanList if c[0] == '#'): check = [channel] + [ban[3:] for ban in chanList[channel]['ban'] if ban.startswith('$j:#')] bans.extend([(channel, ban2check, ban + (checkChan != channel and ' in ' + checkChan or '')) for checkChan in check for ban in chanList[checkChan]['ban'] for ban2check in chanList[channel]['ban'] if (checkChan != channel and not ban.startswith('$j') if ban == ban2check else maskre(ban, ban2check))]) quiets.extend([(channel, quiet2check, quiet) for quiet in chanList[channel]['quiet'] for quiet2check in chanList[channel]['quiet'] if quiet != quiet2check and maskre(quiet, quiet2check)]) return render_template('redundant.html', title='Redundant bans and quiets', bans=bans, quiets=quiets, **userparams()) @app.route('/reload') def reloadchans(): if not 'user' in session: return redirect_login() loadChans() return render_template('chansreloaded.html', title='Chans reloaded', **userparams()) +@app.route('/checkuser/') +def reloadwikiuser(user): + if not 'user' in session: + return redirect_login() + checkuser(user) + return 'checkuser(%r)' % user + def notAuth(): return render_template('notauth.html', title=u'Not authenticated') @app.errorhandler(404) def page_not_found(error): return render_template('page_not_found.html', title=u'Page not found'), 404 def redirect_login(): returnto = request.full_path url = returnto and returnto != '/' and url_for('login', returnto=returnto) or url_for('login') r = redirect(url) return r def dbconn(): global _dbconn if _dbconn[0] + 30 > time.time(): return _dbconn[1] if _dbconn[1]: try: _dbconn[1].close() except Exception as e: print repr(e) connection = oursql.connect(db='s53213__wmopbot', host='tools.labsdb', read_default_file=os.path.expanduser('~/replica.my.cnf'), use_unicode=False) c = connection.cursor() _dbconn = [time.time(), c] return c def query(sql, args=()): c = dbconn() c.execute(sql, args) return c.fetchall() def execute(sql, args=()): c = dbconn() c.execute(sql, args) def loadChans(): global chanList, chanTable, chanLoaded, chanacs chanList = defaultdict(lambda: {'access': {}, 'ban': {}, 'quiet': {}, 'mode': {}, 'exempt': {}, 'config': {}}) for chan, _type, target, args in query('SELECT * FROM list'): chanList[chan].setdefault(_type, {})[target.decode('utf-8', 'replace')] = \ {k: v.decode('utf-8', 'replace').replace('\;', ';') for k, v in (i.split(':', 1) for i in sep.split(args))} stats = query('SELECT st_channel, SUM(st_msg), (SUM(st_users) / COUNT(*)) FROM stats WHERE st_hour > (UNIX_TIMESTAMP() / 3600 - 168) GROUP BY st_channel') stats = {chan: (int(msg), int(usr)) for chan, msg, usr in stats} with open('.online') as f: online = f.read().split('\n')[1:] chanTable = [] for chan in chanList: if chan[0] != '#': continue ops = len([1 for args in chanList[chan]['access'].values() if 'o' in args['flags']]) wmfgc = 'wmfgc' in chanList[chan]['access'] and chanList[chan]['access']['wmfgc'].get('flags', '') or \ chanList[chan]['config'].get('update', {}).get('cserror') or '' bans = len([1 for b in chanList[chan]['ban']]) jbans = ', '.join([b for b in chanList[chan]['ban'] if b.startswith('$j:')]) modes = chanList[chan]['config'].get('cs/irc', {}).get('mode', '?') csflags = chanList[chan]['config'].get('cs/irc', {}).get('csflags', '') msg = chan in stats and stats[chan][0] or chan in online and '0' or '' users = chan in stats and stats[chan][1] or '' sigyn = 'Sigyn' in chanList[chan]['config'] and 'yes' or '' if 'WM' in chanList[chan]['config']: chanTable.append((chan, ops, wmfgc, bans, jbans, modes, csflags, msg, users, sigyn)) chanTable.sort() chanacs = [(c, i[9:]) for c in chanList for i in chanList[c]['access'] if i.startswith('$chanacs:') and 'o' in chanList[c]['access'][i].get('flags', '')] chanLoaded = int(time.time()) loadChans() def userparams(): user = session['user'] chans = query(r"SELECT ls_channel FROM list WHERE ls_type = 'access' AND ls_target = ? AND ls_args RLIKE 'flags:\\w*[Oo]\\w*;'", (user,)) userChans = {i[0] for i in chans} if not user in userType: utype = 'op' if user == 'danilomac': utype = 'dev' elif chanList['#wikimedia-ops']['access'].get(user, {}).get('template') == 'Contact': utype = 'gc' elif query("SELECT 1 FROM user WHERE us_account = ? and us_user LIKE '%@freenode/staff/%' LIMIT 1", (user,)): utype = 'staff' userType[user] = utype else: utype = userType[user] for chan, pull in chanacs: if pull in chanList and user in chanList[pull]['access']: userChans.add(chan) return {'user': user, 'userType': utype, 'userChans': sorted(userChans)} app.add_template_filter(lambda t: t and t.isdigit() and time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(t))) or t, name='ftime') app.add_template_filter(lambda t: t.decode('utf-8', 'replace'), name='utf8') def checkuser(wikiacc): with open('web.log', 'a') as f: f.write('checkuser: %r\n' % userCheck) if not wikiacc in userCheck: + if type(wikiacc) == unicode: + wikiacc = wikiacc.encode('utf-8') userCheck[wikiacc] = [time.time(), None] if not userCheck[wikiacc][1] and not wikiacc in threadQueue: threadQueue.append(wikiacc) if threadQueue and not lock.locked(): thread.start_new_thread(checkuser_thread, ()) def checkuser_thread(): lock.acquire() try: while threadQueue: checkuser_thread2(threadQueue.pop(0)) for u, v in userCheck.items(): if v[0] + 86400 < time.time(): del userCheck[u] finally: lock.release() def checkuser_thread2(user): dbs = None wikis = {} dbtime = [] for s in ('s1', 's2', 's3', 's4', 's5', 's6', 's7'): connection = oursql.connect(host=s + '.labsdb', read_default_file=os.path.expanduser('~/replica.my.cnf'), - read_timeout=10, use_unicode=True, autoreconnect=True, autoping=True) + read_timeout=10, use_unicode=False, autoreconnect=True, autoping=True) c = connection.cursor() if not dbs: c.execute('SELECT dbname, url FROM meta_p.wiki') - dbs = {db[0].encode('utf-8') + '_p': db[1] for db in c.fetchall() if db[1]} + dbs = {db[0].encode('utf-8') + '_p': db[1] for db in c.fetchall() if db[1] and db[0] != 'idwikimedia'} c.execute('SHOW DATABASES') sdbs = [db[0] for db in c.fetchall() if db[0] in dbs] if not 'global' in wikis and 'centralauth_p' in sdbs: c.execute("SELECT gb_expiry FROM centralauth_p.globalblocks WHERE gb_address = ?", (user,)) block = c.fetchall() wikis['global'] = block and 'Global blocked (expiry: %s) ' % expirytime(block[0]) or None for db in sdbs: wikis[dbs[db]] = checkwiki(c, db, user) del dbs[db] c.connection.close() if not dbs: break connection.close() - t = min(d[0] for d in wikis.itervalues() if d) - data = '%s: registered on %s-%s-%s, %s' % (user, t[0:4], t[4:6], t[6:8], wikis.pop('global', None) or '') - wikis = sorted(((w, d) for w, d in wikis.iteritems() if d), key=lambda i:i[1][1], reverse=True) - data += ', '.join('%s (%s)' % (w.split('//')[-1], d[2]) for w, d in wikis[:(len(wikis) > 8 and 6 or 8)]) - if len(wikis) > 8: - data += ', and more %d wikis with the average of %.1f edits per wiki' % (len(wikis) - 6, - float(sum(d[1] for w, d in wikis[6:])) / (len(wikis) - 6)) + if any(wikis.itervalues()): + t = min(d[0] for d in wikis.itervalues() if d) + data = '%s: registered on %s-%s-%s, %s' % (user, t[0:4], t[4:6], t[6:8], wikis.pop('global', None) or '') + wikis = sorted(((w, d) for w, d in wikis.iteritems() if d), key=lambda i:i[1][1], reverse=True) + data += ', '.join('%s (%s)' % (w.split('//')[-1], d[2]) for w, d in wikis[:(len(wikis) > 8 and 6 or 8)]) + if len(wikis) > 8: + data += ', and more %d wikis with the average of %.1f edits per wiki' % (len(wikis) - 6, + float(sum(d[1] for w, d in wikis[6:])) / (len(wikis) - 6)) + else: + data = '%s has no edits in any Wikimedia Project' % user userCheck[user][1] = data id_data = query('SELECT cl_id, cl_status, cl_data FROM cloak WHERE cl_wikiacc = ?', (user,)) for cl_id, cl_status, cl_data in id_data: cl_data = cl_data.split('\n') if len(cl_data) == 1: cl_data.append('') if not cl_data[1] or cl_status == 'new': cl_data[1] = data execute('UPDATE cloak SET cl_data = ? WHERE cl_id = ?', ('\n'.join(cl_data), cl_id)) def checkwiki(c, db, user): c.execute("SELECT user_registration, user_editcount, user_id FROM %s.user WHERE user_name = ?" % db, (user,)) r = c.fetchall() if not r or not r[0][1]: return False regtime, count, user_id = str(r[0][0]), int(r[0][1]), int(r[0][2]) data = '%d edit%s' % (count, count != 1 and 's' or '') c.execute("SELECT ipb_expiry FROM %s.ipblocks_ipindex WHERE ipb_address = ?" % db, (user,)) block = c.fetchall() if block: data += ', blocked (expiry: %s)' % block c.execute("SELECT ug_group FROM %s.user_groups WHERE ug_user = ?" % db, (user_id,)) groups = [g[0] for g in c.fetchall()] if groups: data += ', ' + ', '.join(groups) return regtime, count, data def expirytime(e): return e.isdigit and '%s-%s-%s %s:%s:%s' % (e[0:4], e[4:6], e[6:8], e[8:10], e[10:12], e[12:14]) or e def irclower(s): "IRC lower case: acording RFC 2812, the characters {}|^ are the respective lower case of []\~" return s.lower().replace('{', '[').replace('}', ']').replace('|', '\\').replace('^', '~') mask2re = {'\\': '[\\\\|]', '|': '[\\\\|]', '^': '[~^]', '~': '[~^]', '[': '[[{]', ']': '[]}]', '{': '[[{]', '}': '[]}]', '*': '[^!@]*', '?': '[^!@]', '+': '\\+', '.': '\\.', '(': '\\(', ')': '\\)', '$': '\\$'} def maskre(mask, match=None): "Transforms an IRC mask into a regex pattern" mask = irclower(mask) regex = '' for c in mask: regex += mask2re.get(c, c) try: r = re.compile(regex + '$', re.I) except: r = None if not match: return r return r.match(match) if __name__ == '__main__': with open('.secret') as f: app.secret_key = f.read() if os.uname()[1].startswith('tools-webgrid'): app.config['APPLICATION_ROOT'] = '/wmopbot/' from flup.server.fcgi_fork import WSGIServer WSGIServer(app).run() else: # Run outside Labs for tests (no database) app.run() # print 'This tool does not work outside wmopbot project in Tool Labs' diff --git a/wikiblocks.py b/wikiblocks.py index fcd7cee..647d230 100755 --- a/wikiblocks.py +++ b/wikiblocks.py @@ -1,132 +1,132 @@ #! /usr/bin/python # -*- coding: utf-8 -*- import oursql, os, re, time class dbConn: "Connect to bot database" def __init__(self, host): self.host = host self.connection = None self.connect() self.count = 0 def connect(self): try: self.connection.close() except: pass self.connection = oursql.connect(host=self.host, read_default_file=os.path.expanduser('~/replica.my.cnf'), read_timeout=10, use_unicode=True, autoreconnect=True, autoping=True) self.cursor = self.connection.cursor() def execute(self, sql, params=()): try: self.cursor.execute(sql, params) self.last = time.time() self.count = 0 except oursql.OperationalError: # probably connection lost if self.count > 3: raise Exception('Connection lost more than 3 times') print 'Lost connection', sql, params if self.count > 0: time.sleep(10) self.count += 1 self.connect() self.execute(sql, params) def fetchall(self): return self.cursor.fetchall() def cleanIP(ip): "Clean IP range, e.g. '1.2.3.4/20' => '1.2.0.0/20'" ipv4 = re.match(r'(\d{1,3}\.(?:\*|\d{1,3})\.(?:\*|\d{1,3})\.(?:\*|\d{1,3}))(?:/([1-3]?\d))?', ip) if ipv4: octets = [int(o) for o in ipv4.group(1).split('.')] cdir = ipv4.group(2) and int(ipv4.group(2)) or 32 if cdir > 32: cdir = 32 num = reduce(lambda a, b: a << 8 | b, octets) >> 32 - cdir << 32 - cdir return '.'.join(str(num >> 24 - i * 8 & 255) for i in range(4)) + ('/%d' % cdir if cdir < 32 else '') ipv6 = re.match(r'([0-9a-fA-F]{1,4}(?:::?[0-9a-fA-F]{1,4}){1,7}(?:::)?)(?:/(1?\d{1,2}))?', ip) if ipv6: parts = [[g or '0' for g in part.split(':')] for part in ipv6.group(1).split('::')] groups = parts[0] + (len(parts) == 2 and ['0'] * (8 - len(parts[0]) - len(parts[1])) + parts[1] or []) cdir = ipv6.group(2) and int(ipv6.group(2)) or 128 if cdir > 128: cdir = 128 num = reduce(lambda a, b: a << 16 | b, [int(g, 16) for g in groups]) >> 128 - cdir << 128 - cdir return re.sub(':0(?::0)*$', '::', ':'.join('%x' % (num >> 112 - i * 16 & 65535) for i in range(8)), 1) + \ ('/%d' % cdir if cdir < 128 else '') else: return False def loadBlocks(db): if db == 'ruwiki_p': # ruwiki has 1300k+ IP blocks, jump to not make the db lost the connection log.write('\tjumping\n') return start = time.time() wiki = db == 'centralauth_p' and 'global' or db[:-2] if db == 'centralauth_p': query = "SELECT gb_address FROM %s.globalblocks" elif db in ('jawiki_p', 'nlwiki_p'): # too many /16 blocks query = "SELECT ipb_address FROM %s.ipblocks WHERE ipb_user = 0 AND ipb_address NOT LIKE '%%/16'" else: query = "SELECT ipb_address FROM %s.ipblocks WHERE ipb_user = 0" c.execute(query % db) ips = [i[0] for i in c.fetchall() if i and i[0]] b = 0 for i in ips: ip = cleanIP(i) if not ip: #print 'Not recgnized IP: ' + i continue blockWiki.setdefault(ip, []).append(wiki) b += 1 log.write('\t%d blocks\t%.2f sec\n' % (b, time.time() - start)) log = open('wikiblocks.log', 'w') log.write(time.strftime('%Y-%m-%d %H:%M:%S\n')) print time.strftime('%Y-%m-%d %H:%M:%S') start = time.time() blockWiki = {} # Load blocks data from db replicas for s in ('s1', 's2', 's3', 's4', 's5', 's6', 's7'): c = dbConn(s +'.labsdb') if not 'dbs' in globals(): c.execute('SELECT dbname FROM meta_p.wiki') - dbs = [db[0].encode('utf-8') + '_p' for db in c.fetchall()] + ['centralauth_p'] + dbs = [db[0].encode('utf-8') + '_p' for db in c.fetchall() if db[0] != 'idwikimedia'] + ['centralauth_p'] c.execute('SHOW DATABASES') sdbs = [db[0] for db in c.fetchall()] log.write(s + '\n') for db in sdbs: if db in dbs: log.write(db) loadBlocks(db) dbs.remove(db) c.connection.close() if not dbs: break # Save data in local db start2 = time.time() connection = oursql.connect(db='s53213__wmopbot', host='tools.labsdb', read_default_file=os.path.expanduser('~/replica.my.cnf'), read_timeout=10, use_unicode=True, autoreconnect=True, autoping=True) c = connection.cursor() blocks = sorted(blockWiki) log.write('Total\t%d distinct blocks' % len(blocks)) c.execute('ALTER TABLE wikiblocks DISABLE KEYS') c.execute('TRUNCATE wikiblocks') n = 0 while n < len(blocks): c.execute('INSERT INTO wikiblocks VALUES ' + ','.join('(?, ?)' for b in blocks[n:n + 10000]), tuple(i for x in ((ip, ' '.join(blockWiki[ip])) for ip in blocks[n:n + 10000]) for i in x)) n += 10000 c.execute('ALTER TABLE wikiblocks ENABLE KEYS') connection.close() log.write('\tsaved in %.2f sec\n' % (time.time() - start2)) t = time.time() - start log.write((t >= 60 and '%d min ' % (t / 60) or '') + '%d sec' % (t % 60)) log.close()