diff --git a/Pipfile b/Pipfile
index ebf1b29..0f45f94 100644
--- a/Pipfile
+++ b/Pipfile
@@ -1,15 +1,15 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[dev-packages]
mock = "*"
nose = "*"
[packages]
Flask = "*"
PyMySQL = "*"
[requires]
-python_version = "2.7"
+python_version = "3.7"
diff --git a/app.py b/app.py
index 1b30ce5..a23a0d3 100755
--- a/app.py
+++ b/app.py
@@ -1,199 +1,199 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import os
import re
import time
from os.path import getmtime
from flask import Flask, make_response, render_template, request
import images
from functions import (
get_country_data,
get_country_summary,
get_edition_data,
get_edition_name,
get_event_name,
get_events_data,
get_instance_name,
get_instance_users_data,
get_menu,
get_wikiloves_category_name,
normalize_country_name
)
app = Flask(__name__)
app.debug = True
dbtime = None
def loadDB():
global db, menu, events_data, events_names, country_data, dbtime
mtime = getmtime('db.json')
if dbtime and dbtime == mtime:
return
dbtime = mtime
try:
with open('db.json', 'r') as f:
db = json.load(f)
except IOError:
db = None
menu = get_menu(db)
events_data = get_events_data(db)
- events_names = {slug: get_event_name(slug) for slug in events_data.keys()}
+ events_names = {slug: get_event_name(slug) for slug in list(events_data.keys())}
country_data = get_country_data(db)
loadDB()
@app.route('/')
def index():
countries = get_country_summary(country_data)
- return render_template('mainpage.html', title=u'Wiki Loves Competitions Tools', menu=menu,
+ return render_template('mainpage.html', title='Wiki Loves Competitions Tools', menu=menu,
data=events_data, events_names=events_names, countries=countries)
@app.route('/log')
def logpage():
try:
with open('update.log', 'r') as f:
log = f.read()
timestamp = time.strftime('%H:%M, %d %B %Y', time.strptime(log[:14], '%Y%m%d%H%M%S'))
- log = re.sub(ur'\[\[([^]]+)\]\]', lambda m: u'%s' %
- (m.group(1).replace(u' ', u'_'), m.group(1)), log[15:]).split(u'\n')
+ log = re.sub(r'\[\[([^]]+)\]\]', lambda m: '%s' %
+ (m.group(1).replace(' ', '_'), m.group(1)), log[15:]).split('\n')
except IOError:
log = timestamp = None
- return render_template('log.html', title=u'Update log', menu=menu, time=timestamp, log=log)
+ return render_template('log.html', title='Update log', menu=menu, time=timestamp, log=log)
# All routes are explicit as we cannot just route // as it would also route eg /images/
@app.route('/monuments', defaults={'scope': 'monuments'})
@app.route('/earth', defaults={'scope': 'earth'})
@app.route('/africa', defaults={'scope': 'africa'})
@app.route('/public_art', defaults={'scope': 'public_art'})
@app.route('/science', defaults={'scope': 'science'})
@app.route('/food', defaults={'scope': 'food'})
@app.route('/folklore', defaults={'scope': 'folklore'})
def event_main(scope):
if not db:
return index()
if scope in events_data:
eventName = get_event_name(scope)
- eventData = {scope: {y: v for y, v in events_data[scope].iteritems()}}
+ eventData = {scope: {y: v for y, v in events_data[scope].items()}}
eventData.update(countries={country: country_data[country][event] for country in country_data
for event in country_data[country] if event == scope})
return render_template('eventmain.html', title=eventName, menu=menu, scope=scope, data=eventData)
else:
- return render_template('page_not_found.html', title=u'Event not found', menu=menu)
+ return render_template('page_not_found.html', title='Event not found', menu=menu)
@app.route('//20')
def edition(scope, year):
loadDB()
if not db:
return index()
year = '20' + year
edition_slug = scope + year
if edition_slug in db:
edition_name = get_edition_name(scope, year)
edition_data = get_edition_data(db, edition_slug)
return render_template('edition.html', title=edition_name, menu=menu,
data=edition_data, rickshaw=True)
else:
- return render_template('page_not_found.html', title=u'Edition not found', menu=menu)
+ return render_template('page_not_found.html', title='Edition not found', menu=menu)
@app.route('//20//users')
def users(scope, year, country):
if not db:
return index()
year = '20' + year
country = normalize_country_name(country)
edition_slug = scope + year
if edition_slug in db and country in db[edition_slug]:
instance_name = get_instance_name(scope, year, country)
eventUsers = get_instance_users_data(db, edition_slug, country)
return render_template('users.html', title=instance_name, menu=menu, scope=scope, year=year,
country=country, data=eventUsers, starttime=db[edition_slug][country]['start'])
elif edition_slug in db:
- return render_template('page_not_found.html', title=u'Country not found', menu=menu)
+ return render_template('page_not_found.html', title='Country not found', menu=menu)
else:
- return render_template('page_not_found.html', title=u'Edition not found', menu=menu)
+ return render_template('page_not_found.html', title='Edition not found', menu=menu)
@app.route('//20/')
def instance(scope, year, country):
if not db:
return index()
year = '20' + year
edition_slug = scope + year
category_name = get_wikiloves_category_name(scope, year, country)
country = normalize_country_name(country)
if edition_slug in db and country in db[edition_slug]:
instance_name = get_instance_name(scope, year, country)
instance_daily_data = db[edition_slug][country]['data']
return render_template('instance.html', title=instance_name, menu=menu, category_name=category_name,
daily_data=instance_daily_data, starttime=db[edition_slug][country]['start'])
elif edition_slug in db:
- return render_template('page_not_found.html', title=u'Country not found', menu=menu)
+ return render_template('page_not_found.html', title='Country not found', menu=menu)
else:
- return render_template('page_not_found.html', title=u'Edition not found', menu=menu)
+ return render_template('page_not_found.html', title='Edition not found', menu=menu)
@app.route('/country/')
def country(name):
name = normalize_country_name(name)
if name in country_data:
- return render_template('country.html', title=u'Wiki Loves Competitions in ' + name, menu=menu,
+ return render_template('country.html', title='Wiki Loves Competitions in ' + name, menu=menu,
data=country_data[name], events_names=events_names, country=name)
else:
- return render_template('page_not_found.html', title=u'Country not found', menu=menu)
+ return render_template('page_not_found.html', title='Country not found', menu=menu)
@app.route('/images')
def images_page():
- args = dict(request.args.items())
+ args = dict(list(request.args.items()))
imgs = images.get(args)
if not imgs:
- return render_template('images_not_found.html', menu=menu, title=u'Images not found')
+ return render_template('images_not_found.html', menu=menu, title='Images not found')
backto = [args['event'], args['year']] + ([args['country']] if 'user' in args else [])
- title = u'Images of %s%s %s in %s' % (args['user'] + u' in ' if 'user' in args else u'',
- get_event_name(args['event']),
- args['year'], args['country'])
+ title = 'Images of %s%s %s in %s' % (args['user'] + ' in ' if 'user' in args else '',
+ get_event_name(args['event']),
+ args['year'], args['country'])
return render_template('images.html', menu=menu, title=title, images=imgs, backto=backto)
@app.route('/db.json')
def download():
response = make_response(json.dumps(db))
response.headers["Content-Disposition"] = "attachment; filename=db.json"
response.headers["Content-type"] = "application/json"
return response
@app.template_filter(name='date')
def date_filter(s):
if type(s) == int:
s = str(s)
return '%s-%s-%s' % (s[0:4], s[4:6], s[6:8])
@app.errorhandler(404)
def page_not_found(error):
- return render_template('page_not_found.html', title=u'Page not found', menu=menu), 404
+ return render_template('page_not_found.html', title='Page not found', menu=menu), 404
if __name__ == '__main__':
if os.uname()[1].startswith('tools-webgrid'):
from flup.server.fcgi_fork import WSGIServer
WSGIServer(app).run()
else:
if os.environ.get('LOCAL_ENVIRONMENT', False):
app.run(host='0.0.0.0')
else:
app.run()
diff --git a/commons_database.py b/commons_database.py
index fdee836..892c862 100644
--- a/commons_database.py
+++ b/commons_database.py
@@ -1,57 +1,57 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import time
import pymysql
class DB:
"""
Classe para fazer consultas ao banco de dados
"""
def connect(self):
username = os.environ.get('DB_USERNAME', None)
password = os.environ.get('DB_PASSWORD', None)
host = os.environ.get('DB_HOST', 'commonswiki.analytics.db.svc.eqiad.wmflabs')
self.conn = pymysql.connect(
db='commonswiki_p',
host=host,
user=username,
passwd=password,
read_default_file=os.path.expanduser('~/replica.my.cnf'),
read_timeout=30, charset='utf8', use_unicode=True)
self.conn.ping(True)
def _query(self, *sql):
with self.conn.cursor() as cursor:
cursor.execute(*sql)
return cursor.fetchall()
def query(self, *sql):
"""
Tenta fazer a consulta, reconecta até 10 vezes até conseguir
"""
loops = 0
self.connect()
while True:
try:
return self._query(*sql)
except (AttributeError, pymysql.err.OperationalError):
if loops < 10:
loops += 1
- print 'Erro no DB, esperando %ds antes de tentar de novo' % loops
+ print('Erro no DB, esperando %ds antes de tentar de novo' % loops)
time.sleep(loops)
else:
return self._query(*sql)
break
else:
- print "Uncaught exception when running query"
- print sql
+ print("Uncaught exception when running query")
+ print(sql)
break
self.close_connection()
def close_connection(self):
self.conn.close()
diff --git a/conf/Dockerfile.web b/conf/Dockerfile.web
index 9e811e7..0c337db 100644
--- a/conf/Dockerfile.web
+++ b/conf/Dockerfile.web
@@ -1,5 +1,5 @@
-FROM python:2.7
+FROM python:3.7
WORKDIR /code
ADD requirements.txt /code/
RUN pip install -r requirements.txt
ADD . /code
diff --git a/configuration.py b/configuration.py
index e3efede..accdf99 100644
--- a/configuration.py
+++ b/configuration.py
@@ -1,65 +1,65 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import re
-from urllib import urlopen
+from urllib.request import urlopen
from functions import EVENTS
def reData(txt, year):
"""
Parser para linha da configuração
"""
events = '|'.join(EVENTS)
- regex = ur'''
+ regex = r'''
\s*wl\["(?P%s)"\]\[(?P20\d\d)]\ ?=\ ?\{|
\s*\["(?P[-a-z]+)"\]\ =\ \{\["start"\]\ =\ (?P%s\d{10}),\ \["end"\]\ =\ (?P%s\d\d{10})\}
''' % (events, year, str(year)[:3])
m = re.search(regex, txt, re.X)
return m and m.groupdict()
def re_prefix(txt):
return re.search(r'\s*\["(?P[\w-]+)"\] = "(?P[\w\-\' ]+)"|(?P\})', txt, re.UNICODE)
def get_config_from_commons(page):
api = urlopen('https://commons.wikimedia.org/w/api.php?action=query&format=json&prop=revisions&titles=%s&rvprop=content' % page)
- text = json.loads(api.read())['query']['pages'].values()[0]['revisions'][0]['*']
- return unicode(text)
+ text = list(json.loads(api.read())['query']['pages'].values())[0]['revisions'][0]['*']
+ return str(text)
def parse_config(text):
data, event, prefixes = {}, None, {}
- lines = iter(text.split(u'\n'))
+ lines = iter(text.split('\n'))
for line in lines:
m = re_prefix(line)
if prefixes and m and m.group('close'):
break
elif m and m.group('prefix'):
prefixes[m.group('prefix')] = m.group('name')
for line in lines:
- g = reData(line, event[-4:] if event else ur'20\d\d')
+ g = reData(line, event[-4:] if event else r'20\d\d')
if not g:
continue
if g['event']:
event = g['event'] + g['year']
data[event] = {}
elif g['country'] and event:
if g['country'] not in prefixes:
# updateLog.append(u'Unknown prefix: ' + g['country'])
continue
data[event][prefixes[g['country']]] = {'start': int(g['start']), 'end': int(g['end'])}
- return {name: config for name, config in data.items() if config}
+ return {name: config for name, config in list(data.items()) if config}
def getConfig(page):
"""
Lê a configuração da página de configuração no Commons
"""
text = get_config_from_commons(page)
return parse_config(text)
diff --git a/database.py b/database.py
index 9fe1bdb..9504d8a 100755
--- a/database.py
+++ b/database.py
@@ -1,187 +1,187 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import io
import json
import time
from commons_database import DB
from configuration import getConfig
from functions import get_wikiloves_category_name
updateLog = []
-dbquery = u'''SELECT
+dbquery = '''SELECT
img_timestamp,
img_name IN (SELECT DISTINCT gil_to FROM globalimagelinks) AS image_in_use,
COALESCE(user.user_name, actor.actor_id) as name,
COALESCE(user_registration, "20050101000000") as user_registration
FROM (SELECT
cl_to,
cl_from
FROM categorylinks
WHERE cl_to = %s AND cl_type = 'file') cats
INNER JOIN page ON cl_from = page_id
INNER JOIN image ON page_title = img_name
LEFT JOIN oldimage ON image.img_name = oldimage.oi_name AND oldimage.oi_timestamp = (SELECT MIN(o.oi_timestamp) FROM oldimage o WHERE o.oi_name = image.img_name)
LEFT JOIN actor ON actor.actor_id = COALESCE(oldimage.oi_actor, image.img_actor)
LEFT JOIN user ON user.user_id = actor.actor_user
'''
def getData(name, data):
"""
Coleta dados do banco de dados e processa
"""
default_starttime = min(data[c]['start'] for c in data if 'start' in data[c])
default_endtime = max(data[c]['end'] for c in data if 'end' in data[c])
result_data = {}
- for country_name, country_config in data.iteritems():
+ for country_name, country_config in data.items():
event = name[0:-4].title()
year = name[-4:]
cat = get_wikiloves_category_name(event, year, country_name)
if name == 'monuments2010':
- cat = u'Images_from_Wiki_Loves_Monuments_2010'
+ cat = 'Images_from_Wiki_Loves_Monuments_2010'
start_time = country_config.get('start', default_starttime)
end_time = country_config.get('end', default_endtime)
country_data = get_country_data(cat, start_time, end_time)
if country_data:
result_data[country_name] = country_data
else:
- updateLog.append(u'%s in %s is configured, but no file was found in [[Category:%s]]' %
- (name, country_name, cat.replace(u'_', u' ')))
+ updateLog.append('%s in %s is configured, but no file was found in [[Category:%s]]' %
+ (name, country_name, cat.replace('_', ' ')))
return result_data
def get_country_data(category, start_time, end_time):
country_data = {}
dbData = get_data_for_category(category)
if not dbData:
return None
daily_data = {} # data: {timestamp_day0: {'images': n, 'joiners': n}, timestamp_day1: ...}
user_data = {} # users: {'user1': {'count': n, 'usage': n, 'reg': timestamp},...}
discarded_counter = 0
for timestamp, usage, user, user_reg in dbData:
# Desconsidera timestamps fora do período da campanha
if not start_time <= timestamp <= end_time:
discarded_counter += 1
continue
# Conta imagens por dia
day = str(timestamp)[0:8]
if day not in daily_data:
daily_data[day] = {'images': 0, 'joiners': 0, 'newbie_joiners': 0}
daily_data[day]['images'] += 1
if user not in user_data:
daily_data[day]['joiners'] += 1
if user_reg > start_time:
daily_data[day]['newbie_joiners'] += 1
user_data[user] = {'count': 0, 'usage': 0, 'reg': user_reg}
user_data[user]['count'] += 1
if usage:
user_data[user]['usage'] += 1
country_data.update(
{'data': daily_data, 'users': user_data})
country_data['usercount'] = len(user_data)
- country_data['count'] = sum(u['count'] for u in user_data.itervalues())
- country_data['usage'] = sum(u['usage'] for u in user_data.itervalues())
- country_data['userreg'] = len([user for user in user_data.itervalues() if user['reg'] > start_time])
+ country_data['count'] = sum(u['count'] for u in user_data.values())
+ country_data['usage'] = sum(u['usage'] for u in user_data.values())
+ country_data['userreg'] = len([user for user in user_data.values() if user['reg'] > start_time])
country_data['category'] = category
country_data['start'] = start_time
country_data['end'] = end_time
if discarded_counter:
- updateLog.append(u'%s images discarded as out of bounds in [[Category:%s]]' %
- (discarded_counter, category.replace(u'_', u' ')))
+ updateLog.append('%s images discarded as out of bounds in [[Category:%s]]' %
+ (discarded_counter, category.replace('_', ' ')))
return country_data
def get_data_for_category(category_name):
"""Query the database for a given category
Return: Tuple of tuples (, , , )
(20140529121626, False, u'Example', 20140528235032)
"""
query_data = commonsdb.query(dbquery, (category_name,))
dbData = tuple(convert_database_record(record) for record in query_data)
return dbData
def convert_database_record(record):
(timestamp, usage, user, user_reg) = record
return (
int(timestamp),
bool(usage),
- user.decode('utf-8'),
+ user,
int(user_reg or 0)
)
def write_database_as_json(db):
with open('db.json', 'w') as f:
json.dump(db, f)
def update_event_data(event_slug, event_configuration, db):
start = time.time()
event_data = getData(event_slug, event_configuration)
db[event_slug] = event_data
write_database_as_json(db)
log = 'Saved %s: %dsec, %d countries, %d uploads' % \
(event_slug, time.time() - start, len(event_data), sum(event_data[c].get('count', 0) for c in event_data))
- print log
+ print(log)
updateLog.append(log)
return db
if __name__ == '__main__':
from argparse import ArgumentParser
description = "Update the database"
parser = ArgumentParser(description=description)
parser.add_argument("events", nargs='*',
metavar="EVENTS",
help='A list of events to update')
args = parser.parse_args()
- print "Fetching configuration..."
- config = getConfig(u'Module:WL_data')
+ print("Fetching configuration...")
+ config = getConfig('Module:WL_data')
try:
with open('db.json', 'r') as f:
db = json.load(f)
except Exception as e:
- print u'Erro ao abrir db.json:', repr(e)
+ print('Erro ao abrir db.json:', repr(e))
db = {}
- print "Found %s events in the configuration." % len(config)
+ print("Found %s events in the configuration." % len(config))
commonsdb = DB()
if args.events:
- print "Updating only %s event(s): %s." % (len(args.events), ', '.join(args.events))
+ print("Updating only %s event(s): %s." % (len(args.events), ', '.join(args.events)))
for event_name in args.events:
event_configuration = config.get(event_name)
if event_configuration:
- print "Fetching data for %s..." % event_name
+ print("Fetching data for %s..." % event_name)
db = update_event_data(event_name, event_configuration, db)
else:
- print "Invalid event: %s" % event_name
+ print("Invalid event: %s" % event_name)
else:
- print "Updating all %s events." % len(config)
- for (event_name, event_configuration) in config.iteritems():
- print "Fetching data for %s..." % event_name
+ print("Updating all %s events." % len(config))
+ for (event_name, event_configuration) in config.items():
+ print("Fetching data for %s..." % event_name)
db = update_event_data(event_name, event_configuration, db)
if updateLog:
with io.open('update.log', 'w', encoding='utf-8') as f:
f.write(time.strftime('%Y%m%d%H%M%S') + '\n' + '\n'.join(updateLog))
diff --git a/functions.py b/functions.py
index 534e43e..79bb819 100644
--- a/functions.py
+++ b/functions.py
@@ -1,155 +1,155 @@
# -*- coding: utf-8 -*-
EVENTS = [
'earth',
'monuments',
'africa',
'public_art',
'science',
'food',
'folklore',
]
def get_country_data(db):
country_data = {}
for edition_slug in db:
scope_slug = edition_slug[:-4]
year = edition_slug[-4:]
for country in db[edition_slug]:
country_data.setdefault(country, {}).setdefault(scope_slug, {}).update({year: {
'count': db[edition_slug][country]['count'],
'usercount': db[edition_slug][country]['usercount'],
'usage': db[edition_slug][country]['usage'],
'userreg': db[edition_slug][country]['userreg']}
}
)
return country_data
def get_events_data(db):
return {
name: {
e[-4:]: {
'count': sum(db[e][c]['count'] for c in db[e]),
'usercount': sum(db[e][c]['usercount'] for c in db[e]),
'userreg': sum(db[e][c]['userreg'] for c in db[e]),
'usage': sum(db[e][c]['usage'] for c in db[e]),
'country_count': len(db[e])
}
for e in db if e[:-4] == name
} for name in set(e[:-4] for e in db)
}
def get_edition_data(db, edition_slug):
return {
country: {
field: db[edition_slug][country][field]
for field in db[edition_slug][country] if field != 'users'
} for country in db[edition_slug]
}
def get_instance_users_data(db, edition_slug, country):
return sorted(
- db[edition_slug][country]['users'].items(),
+ list(db[edition_slug][country]['users'].items()),
key=lambda i: (i[1]['count'], i[0]), reverse=True)
def get_menu(db):
return {
name: sorted(e[-4:] for e in db if e[:-4] == name)
for name in set(e[:-4] for e in db)
}
def get_country_summary(country_data):
return {c: [(sorted(country_data[c][event].keys())if event in country_data[c] else None)
for event in EVENTS]
for c in country_data}
def normalize_country_name(country_name):
return country_name.replace('_', ' ')
def get_event_name(event_slug):
"""
Generate a name from the label.
Returns title case with underscore replaced.
"""
- default = u'Wiki Loves %s' % event_slug.replace('_', ' ').title()
+ default = 'Wiki Loves %s' % event_slug.replace('_', ' ').title()
return event_exceptions.get(event_slug.lower(), default)
def get_edition_name(scope_slug, year):
- default = u'%s %s' % (get_event_name(scope_slug), year)
+ default = '%s %s' % (get_event_name(scope_slug), year)
return edition_exceptions.get((scope_slug.lower(), str(year)), default)
def get_instance_name(scope_slug, year, country):
- return u'%s %s in %s' % (get_event_name(scope_slug), year, country)
+ return '%s %s in %s' % (get_event_name(scope_slug), year, country)
def get_wikiloves_category_name(event_slug, year, country):
if (event_slug, year, country) in special_exceptions:
return special_exceptions[(event_slug, year, country)]
edition = get_edition_name(event_slug, year)
template = get_event_category_template()
country_name = catExceptions.get(country, country)
- return template.format(edition=edition, country=country_name).replace(' ', u'_')
+ return template.format(edition=edition, country=country_name).replace(' ', '_')
def get_event_category_template():
- return u'Images_from_{edition}_in_{country}'
+ return 'Images_from_{edition}_in_{country}'
event_exceptions = {
- u'science': 'Wiki Science Competition',
+ 'science': 'Wiki Science Competition',
}
catExceptions = {
- u'Armenia': u'Armenia_&_Nagorno-Karabakh',
- u'Netherlands': u'the_Netherlands',
- u'Central African Republic': u'the_Central_African_Republic',
- u'Comoros': u'the_Comoros',
- u'Czech Republic': u'the_Czech_Republic',
- u'Democratic Republic of the Congo': u'the_Democratic_Republic_of_the_Congo',
- u'Republic of the Congo': u'the_Republic_of_the_Congo',
- u'Dutch Caribbean': u'the_Dutch_Caribbean',
- u'Philippines': u'the_Philippines',
- u'Seychelles': u'the_Seychelles',
- u'United Arab Emirates': u'the_United_Arab_Emirates',
- u'United Kingdom': u'the_United_Kingdom',
- u'United States': u'the_United_States'
+ 'Armenia': 'Armenia_&_Nagorno-Karabakh',
+ 'Netherlands': 'the_Netherlands',
+ 'Central African Republic': 'the_Central_African_Republic',
+ 'Comoros': 'the_Comoros',
+ 'Czech Republic': 'the_Czech_Republic',
+ 'Democratic Republic of the Congo': 'the_Democratic_Republic_of_the_Congo',
+ 'Republic of the Congo': 'the_Republic_of_the_Congo',
+ 'Dutch Caribbean': 'the_Dutch_Caribbean',
+ 'Philippines': 'the_Philippines',
+ 'Seychelles': 'the_Seychelles',
+ 'United Arab Emirates': 'the_United_Arab_Emirates',
+ 'United Kingdom': 'the_United_Kingdom',
+ 'United States': 'the_United_States'
}
edition_exceptions = {
("science", "2015"): 'European_Science_Photo_Competition_2015',
}
special_exceptions = {
("Monuments", "2022", "Austria"): 'Media_from_WikiDaheim_2022_in_Austria/Cultural_heritage_monuments',
("Monuments", "2021", "Austria"): 'Media_from_WikiDaheim_2021_in_Austria/Cultural_heritage_monuments',
("Monuments", "2020", "Austria"): 'Media_from_WikiDaheim_2020_in_Austria/Cultural_heritage_monuments',
("Monuments", "2019", "Austria"): 'Media_from_WikiDaheim_2019_in_Austria/Cultural_heritage_monuments',
("Monuments", "2018", "Austria"): 'Media_from_WikiDaheim_2018_in_Austria/Cultural_heritage_monuments',
("Monuments", "2017", "Austria"): 'Media_from_WikiDaheim_2017_in_Austria/Cultural_heritage_monuments',
("Monuments", "2022", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2022_in_Armenia',
("Monuments", "2021", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2021_in_Armenia',
("Monuments", "2020", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2020_in_Armenia',
("Monuments", "2019", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2019_in_Armenia',
("Monuments", "2013", "Armenia"): 'Images_from_Wiki_Loves_Monuments_2013_in_Armenia',
("Earth", "2020", "Armenia"): 'Images_from_Wiki_Loves_Earth_2020_in_Armenia',
("Earth", "2021", "Armenia"): 'Images_from_Wiki_Loves_Earth_2021_in_Armenia',
("Earth", "2021", "United Arab Emirates"): 'Images_from_Wiki_Loves_Earth_2021_in_United_Arab_Emirates',
("Earth", "2022", "United Arab Emirates"): 'Images_from_Wiki_Loves_Earth_2021_in_United_Arab_Emirates',
("Science", "2015", "Armenia"): 'Images_from_European_Science_Photo_Competition_2015_in_Armenia',
("Science", "2013", "Estonia"): 'Images_from_Teadusfoto_2013',
("Science", "2012", "Estonia"): 'Images_from_Teadusfoto_2012',
("Science", "2011", "Estonia"): 'Images_from_Teadusfoto_2011',
}
diff --git a/images.py b/images.py
index 80baac1..2e220b4 100644
--- a/images.py
+++ b/images.py
@@ -1,61 +1,61 @@
# -*- coding: utf-8 -*-
from commons_database import DB
from functions import get_wikiloves_category_name, normalize_country_name
def makeQuery(args):
- if u'event' in args and u'year' in args and u'country' in args:
+ if 'event' in args and 'year' in args and 'country' in args:
country = normalize_country_name(args['country'])
category = get_wikiloves_category_name(args['event'].title(), args['year'], country)
queryArgs = (category,)
else:
return
start = 'start' in args and args.get('start').isdigit() and int(args.get('start')) or 0
params = {}
- params['user'] = u' AND img_user_text = ?' if u'user' in args else u''
+ params['user'] = ' AND img_user_text = ?' if 'user' in args else ''
if params['user']:
queryArgs += (args['user'].replace('_', ' '),)
- params['start'] = ' OFFSET ' + str(args.get('start')) if start else u''
+ params['start'] = ' OFFSET ' + str(args.get('start')) if start else ''
params['mb'] = minmax(args.get('minmb'), args.get('maxmb'), ' AND img_size', lambda n: int(n) * 1048576)
params['mp'] = minmax(args.get('minmp'), args.get('maxmp'), ' HAVING pixels', lambda n: int(n) * 1000000)
params['timestamp'] = minmax(args.get('from'), args.get('until'), ' AND img_timestamp', lambda n: len(n) == 14 and n)
- return (u'''SELECT
+ return ('''SELECT
img_name,
SUBSTR(MD5(img_name), 1, 2),
img_width,
img_height,
(img_width * img_height) pixels,
img_size,
img_timestamp
FROM categorylinks
INNER JOIN page ON cl_from = page_id
INNER JOIN image ON page_title = img_name
WHERE cl_to = %s AND cl_type = 'file' AND img_major_mime = 'image'{user}{timestamp}{mb}{mp}
ORDER BY pixels DESC
LIMIT 201{start}'''.format(**params), queryArgs)
def get(args):
sql = makeQuery(args)
if not sql:
return
commonsdb = DB()
data = commonsdb.query(*sql)
return [(i[0].decode('utf-8'), i[1], int(i[2]), int(i[3]), i[4], i[5], i[6]) for i in data]
def minmax(pmin, pmax, prefix, func=None):
pmin = (func(pmin) if func else pmin) if pmin and pmin.isdigit() else ''
pmax = (func(pmax) if func else pmax) if pmax and pmax.isdigit() else ''
if pmin:
if pmax:
expr = ' BETWEEN {} AND {}'.format(pmin, pmax)
else:
expr = ' >= {}'.format(m[0]) # noqa
else:
if pmin:
expr = ' <= {}'.format(m[1]) # noqa
else:
expr = ''
return expr and prefix + expr
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index 2ac413b..c0f3b22 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -1,121 +1,121 @@
# -*- coding: utf-8 -*-
"""Unit tests for configuration.py."""
import unittest
import configuration
class TestReData(unittest.TestCase):
def test_reData_event_line(self):
input_data = 'wl["monuments"][2010] = {'
result = configuration.reData(input_data, 2014)
expected = {
- u'country': None,
- u'year': '2010',
- u'end': None,
- u'event': 'monuments',
- u'start': None
+ 'country': None,
+ 'year': '2010',
+ 'end': None,
+ 'event': 'monuments',
+ 'start': None
}
self.assertEqual(result, expected)
def test_reData_event_line_public_art(self):
input_data = 'wl["public_art"][2012] = {'
result = configuration.reData(input_data, 2014)
expected = {
- u'country': None,
- u'year': '2012',
- u'end': None,
- u'event': 'public_art',
- u'start': None
+ 'country': None,
+ 'year': '2012',
+ 'end': None,
+ 'event': 'public_art',
+ 'start': None
}
self.assertEqual(result, expected)
def test_reData_country_line(self):
input_data = '''
["az"] = {["start"] = 20170430200000, ["end"] = 20170531195959},
'''
result = configuration.reData(input_data, 2017)
expected = {
- u'country': 'az',
- u'year': None,
- u'end': '20170531195959',
- u'event': None,
- u'start': '20170430200000'
+ 'country': 'az',
+ 'year': None,
+ 'end': '20170531195959',
+ 'event': None,
+ 'start': '20170430200000'
}
self.assertEqual(result, expected)
class TestRePrefix(unittest.TestCase):
def test_re_prefix_match_ascii_line(self):
- self.assertIsNotNone(configuration.re_prefix(u' ["az"] = "Azerbaijan",'))
+ self.assertIsNotNone(configuration.re_prefix(' ["az"] = "Azerbaijan",'))
def test_re_prefix_match_ascii_line_with_space(self):
- self.assertIsNotNone(configuration.re_prefix(u' ["gq"] = "Equatorial Guinea",'))
+ self.assertIsNotNone(configuration.re_prefix(' ["gq"] = "Equatorial Guinea",'))
def test_re_prefix_match_ascii_line_with_dash(self):
- self.assertIsNotNone(configuration.re_prefix(u' ["gw"] = "Guinea-Bissau",'))
+ self.assertIsNotNone(configuration.re_prefix(' ["gw"] = "Guinea-Bissau",'))
def test_re_prefix_match_ascii_line_with_accents(self):
- self.assertIsNotNone(configuration.re_prefix(u' ["re"] = "Réunion",'))
+ self.assertIsNotNone(configuration.re_prefix(' ["re"] = "Réunion",'))
def test_re_prefix_match_ascii_line_with_apostrophe(self):
- self.assertIsNotNone(configuration.re_prefix(u' ["ci"] = "Côte d\'Ivoire",'))
+ self.assertIsNotNone(configuration.re_prefix(' ["ci"] = "Côte d\'Ivoire",'))
class TestParseConfig(unittest.TestCase):
def test_parse_config_empty(self):
config = ''
result = configuration.parse_config(config)
expected = {}
- self.assertEquals(result, expected)
+ self.assertEqual(result, expected)
def test_parse_config(self):
config = '''
wl["prefixes"] = {
["az"] = "Azerbaijan",
["gw"] = "Guinea-Bissau"
}
wl["monuments"][2017] = {
["az"] = {["start"] = 20170430200000, ["end"] = 20170531195959},
["gw"] = {["start"] = 20170430200000, ["end"] = 20170531195959},
}
wl["monuments"][2018] = {
["az"] = {["start"] = 20180430200000, ["end"] = 20180531195959},
["gw"] = {["start"] = 20180430200000, ["end"] = 20180531195959},
}
'''
result = configuration.parse_config(config)
expected = {
- u'monuments2017': {
- u'Azerbaijan': {
+ 'monuments2017': {
+ 'Azerbaijan': {
'start': 20170430200000,
'end': 20170531195959,
},
- u'Guinea-Bissau': {
+ 'Guinea-Bissau': {
'start': 20170430200000,
'end': 20170531195959,
},
},
- u'monuments2018': {
- u'Azerbaijan': {
+ 'monuments2018': {
+ 'Azerbaijan': {
'start': 20180430200000,
'end': 20180531195959,
},
- u'Guinea-Bissau': {
+ 'Guinea-Bissau': {
'start': 20180430200000,
'end': 20180531195959,
},
}
}
- self.assertEquals(result, expected)
+ self.assertEqual(result, expected)
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_database.py b/tests/test_database.py
index 5b469ba..ad4c8e8 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -1,171 +1,171 @@
# -*- coding: utf-8 -*-
"""Unit tests for database.py."""
import unittest
import mock
import database
class TestConvertDatabaseRecord(unittest.TestCase):
def test_convert_database_record(self):
record = ('20140523121626', 'False', 'Bob', '20130523235032')
result = database.convert_database_record(record)
- expected = (20140523121626, True, u'Bob', 20130523235032)
- self.assertEquals(result, expected)
+ expected = (20140523121626, True, 'Bob', 20130523235032)
+ self.assertEqual(result, expected)
class TestGetDataMixin(unittest.TestCase):
def setUp(self):
patcher = mock.patch('database.get_data_for_category', autospec=True)
self.mock_get_data_for_category = patcher.start()
self.mock_get_data_for_category.return_value = (
- (20140523121626, False, u'Bob', 20130523235032),
- (20140523121626, False, u'Alice', 20140528235032),
- (20140529121626, False, u'Alice', 20140528235032),
- (20140530121626, False, u'Alice', 20140528235032),
+ (20140523121626, False, 'Bob', 20130523235032),
+ (20140523121626, False, 'Alice', 20140528235032),
+ (20140529121626, False, 'Alice', 20140528235032),
+ (20140530121626, False, 'Alice', 20140528235032),
)
self.addCleanup(patcher.stop)
self.expected_timestamp_data = {
'20140523': {
'images': 2,
'joiners': 2,
'newbie_joiners': 1
},
'20140529': {
'images': 1,
'joiners': 0,
'newbie_joiners': 0
},
'20140530': {
'images': 1,
'joiners': 0,
'newbie_joiners': 0
}
}
self.images_count = 4
self.usercount = 2
self.userreg = 1
self.usage = 0
self.user_data = {
- u'Alice': {
+ 'Alice': {
'count': 3,
'reg': 20140528235032,
'usage': 0
},
- u'Bob': {
+ 'Bob': {
'count': 1,
'reg': 20130523235032,
'usage': 0
}
}
class TestGetData(TestGetDataMixin):
def test_GetData(self):
competition_config = {
- u'Brazil': {'start': 20140501030000, 'end': 20140601025959},
+ 'Brazil': {'start': 20140501030000, 'end': 20140601025959},
}
result = database.getData("Dumplings2014", competition_config)
expected = {
- u'Brazil': {
+ 'Brazil': {
'count': self.images_count,
'usercount': self.usercount,
'start': 20140501030000,
'userreg': self.userreg,
'data': self.expected_timestamp_data,
'users': self.user_data,
'usage': self.usage,
- 'category': u'Images_from_Wiki_Loves_Dumplings_2014_in_Brazil',
+ 'category': 'Images_from_Wiki_Loves_Dumplings_2014_in_Brazil',
'end': 20140601025959
}
}
- self.assertEquals(result, expected)
+ self.assertEqual(result, expected)
class TestGetCountryData(TestGetDataMixin):
def test_get_country_data(self):
- category = u'Images_from_Wiki_Loves_Dumplings_2014_in_Brazil'
+ category = 'Images_from_Wiki_Loves_Dumplings_2014_in_Brazil'
result = database.get_country_data(category, 20140501030000, 20140601025959)
expected = {
'count': self.images_count,
'usercount': self.usercount,
'start': 20140501030000,
'userreg': self.userreg,
'data': self.expected_timestamp_data,
'users': self.user_data,
'usage': self.usage,
'category': category,
'end': 20140601025959
}
self.mock_get_data_for_category.assert_called_once_with(category)
- self.assertEquals(result, expected)
+ self.assertEqual(result, expected)
class TestUpdateEventData(TestGetDataMixin):
def setUp(self):
super(self.__class__, self).setUp()
patcher = mock.patch('database.write_database_as_json', autospec=True)
self.mock_write_database_as_json = patcher.start()
self.addCleanup(patcher.stop)
def test_udpate_event_data(self):
self.maxDiff = None
- event_name = u'dumplings2014'
+ event_name = 'dumplings2014'
event_configuration = {
- u'Azerbaijan': {
+ 'Azerbaijan': {
'start': 20140430200000,
'end': 20140531195959,
},
- u'Guinea-Bissau': {
+ 'Guinea-Bissau': {
'start': 20140430200000,
'end': 20140531195959,
},
}
db = {}
result = database.update_event_data(event_name, event_configuration, db)
expected_base = {
'count': self.images_count,
'usercount': self.usercount,
'start': 20140430200000,
'userreg': self.userreg,
'data': self.expected_timestamp_data,
'users': self.user_data,
'usage': self.usage,
'end': 20140531195959
}
expected_az = expected_base.copy()
expected_az.update({
- 'category': u'Images_from_Wiki_Loves_Dumplings_2014_in_Azerbaijan',
+ 'category': 'Images_from_Wiki_Loves_Dumplings_2014_in_Azerbaijan',
})
expected_gb = expected_base.copy()
expected_gb.update({
- 'category': u'Images_from_Wiki_Loves_Dumplings_2014_in_Guinea-Bissau',
+ 'category': 'Images_from_Wiki_Loves_Dumplings_2014_in_Guinea-Bissau',
})
expected = {
- u'dumplings2014': {
- u'Azerbaijan': expected_az,
- u'Guinea-Bissau': expected_gb,
+ 'dumplings2014': {
+ 'Azerbaijan': expected_az,
+ 'Guinea-Bissau': expected_gb,
}
}
- self.assertEquals(result, expected)
+ self.assertEqual(result, expected)
self.mock_write_database_as_json.assert_called_once_with(expected)
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_functions.py b/tests/test_functions.py
index b9957ed..54d2f49 100644
--- a/tests/test_functions.py
+++ b/tests/test_functions.py
@@ -1,410 +1,410 @@
# -*- coding: utf-8 -*-
"""Unit tests for functions.py."""
import json
import os
import unittest
import functions
class TestGetWikilovesCategoryName(unittest.TestCase):
def test_get_wikiloves_category_name(self):
result = functions.get_wikiloves_category_name("Earth", "2016", "France")
- expected = u'Images_from_Wiki_Loves_Earth_2016_in_France'
+ expected = 'Images_from_Wiki_Loves_Earth_2016_in_France'
self.assertEqual(result, expected)
def test_get_wikiloves_category_name_using_exception(self):
result = functions.get_wikiloves_category_name("Earth", "2016", "Netherlands")
- expected = u'Images_from_Wiki_Loves_Earth_2016_in_the_Netherlands'
+ expected = 'Images_from_Wiki_Loves_Earth_2016_in_the_Netherlands'
self.assertEqual(result, expected)
def test_get_wikiloves_category_name_using_special_exception(self):
result = functions.get_wikiloves_category_name("Monuments", "2017", "Austria")
- expected = u'Media_from_WikiDaheim_2017_in_Austria/Cultural_heritage_monuments'
+ expected = 'Media_from_WikiDaheim_2017_in_Austria/Cultural_heritage_monuments'
self.assertEqual(result, expected)
def test_get_wikiloves_category_name_using_event_exception(self):
result = functions.get_wikiloves_category_name("Science", "2017", "Estonia")
- expected = u'Images_from_Wiki_Science_Competition_2017_in_Estonia'
+ expected = 'Images_from_Wiki_Science_Competition_2017_in_Estonia'
self.assertEqual(result, expected)
def test_get_wikiloves_category_name_using_edition_exception(self):
result = functions.get_wikiloves_category_name("Science", "2015", "Estonia")
- expected = u'Images_from_European_Science_Photo_Competition_2015_in_Estonia'
+ expected = 'Images_from_European_Science_Photo_Competition_2015_in_Estonia'
self.assertEqual(result, expected)
class TestGetEventName(unittest.TestCase):
def test_get_event_name_wikiloves(self):
data = {
'earth': 'Wiki Loves Earth',
'africa': 'Wiki Loves Africa',
'monuments': 'Wiki Loves Monuments',
'monuments': 'Wiki Loves Monuments',
}
- for (event_slug, event_name) in data.items():
+ for (event_slug, event_name) in list(data.items()):
result = functions.get_event_name(event_slug)
self.assertEqual(result, event_name)
def test_get_event_name_wikiloves_several_words(self):
result = functions.get_event_name('public_art')
expected = 'Wiki Loves Public Art'
self.assertEqual(result, expected)
def test_get_event_name_wikiloves_exception(self):
result = functions.get_event_name('science')
expected = 'Wiki Science Competition'
self.assertEqual(result, expected)
class TestGetEditionName(unittest.TestCase):
def test_get_edition_name_classic(self):
result = functions.get_edition_name('monuments', 2016)
expected = 'Wiki Loves Monuments 2016'
self.assertEqual(result, expected)
def test_get_edition_name_several_words(self):
result = functions.get_edition_name('public_art', 2016)
expected = 'Wiki Loves Public Art 2016'
self.assertEqual(result, expected)
def test_get_edition_name_exception(self):
result = functions.get_edition_name('science', 2015)
expected = 'European_Science_Photo_Competition_2015'
self.assertEqual(result, expected)
class TestNormalizeCountryName(unittest.TestCase):
def test_normalize_country_name_one_word(self):
result = functions.normalize_country_name('Albania')
expected = 'Albania'
self.assertEqual(result, expected)
def test_normalize_country_name_two_words_with_underscores(self):
result = functions.normalize_country_name('United_States')
expected = 'United States'
self.assertEqual(result, expected)
def test_normalize_country_name_two_words_with_spaces(self):
result = functions.normalize_country_name('United States')
expected = 'United States'
self.assertEqual(result, expected)
def test_normalize_country_name_three_words_with_underscores(self):
result = functions.normalize_country_name('United_Arab_Emirates')
expected = 'United Arab Emirates'
self.assertEqual(result, expected)
def test_normalize_country_name_three_words_with_spaces(self):
result = functions.normalize_country_name('United Arab Emirates')
expected = 'United Arab Emirates'
self.assertEqual(result, expected)
class TestGetCountrySummary(unittest.TestCase):
def test_get_country_summary(self):
country_data = {
"Turkey": {
"earth": {
"2015": {
"count": 5,
"usage": 0,
"userreg": 0,
"usercount": 1
}
},
"monuments": {
"2016": {
"count": 5,
"usage": 0,
"userreg": 0,
"usercount": 1
},
"2017": {
"count": 8,
"usage": 0,
"userreg": 0,
"usercount": 1
}
}
},
"Panama": {
"earth": {
"2016": {
"count": 26,
"usage": 0,
"userreg": 2,
"usercount": 2
}
},
"monuments": {
"2016": {
"count": 22,
"usage": 0,
"userreg": 2,
"usercount": 2
}
}
},
"Benin": {
"africa": {
"2014": {
"count": 5,
"usage": 0,
"userreg": 0,
"usercount": 1
}
}
}
}
result = functions.get_country_summary(country_data)
expected = {
'Benin': [None, None, ['2014'], None, None, None, None],
'Panama': [['2016'], ['2016'], None, None, None, None, None],
'Turkey': [['2015'], ['2016', '2017'], None, None, None, None, None]
}
self.assertEqual(result, expected)
class TestProcessDataMixin(unittest.TestCase):
def setUp(self):
current_path = os.path.abspath(os.path.curdir)
data_file = os.path.join(current_path, 'conf/db.dump.json')
self.data = json.load(open(data_file, 'r'))
class TestProcessData(TestProcessDataMixin):
def test_get_country_data(self):
result = functions.get_country_data(self.data)
expected = {
- u'Austria': {
- u'public_art': {
- u'2013': {
+ 'Austria': {
+ 'public_art': {
+ '2013': {
'count': 5,
'usage': 0,
'usercount': 1,
'userreg': 0
}
}
},
- u'Benin': {
- u'africa': {
- u'2014': {
+ 'Benin': {
+ 'africa': {
+ '2014': {
'count': 5,
'usage': 0,
'usercount': 1,
'userreg': 0
}
}
},
- u'Estonia': {
- u'science': {
- u'2017': {
+ 'Estonia': {
+ 'science': {
+ '2017': {
'count': 9,
'usage': 0,
'usercount': 1,
'userreg': 0
}
}
},
- u'India': {
- u'food': {
- u'2017': {
+ 'India': {
+ 'food': {
+ '2017': {
'count': 9,
'usage': 0,
'usercount': 1,
'userreg': 0
}
},
- u'folklore': {
- u'2022': {
+ 'folklore': {
+ '2022': {
'count': 9,
'usage': 0,
'usercount': 1,
'userreg': 0
}
}
},
- u'Panama': {
- u'earth': {
- u'2015': {
+ 'Panama': {
+ 'earth': {
+ '2015': {
'count': 26,
'usage': 0,
'usercount': 2,
'userreg': 2
}
},
- u'monuments': {
- u'2016': {
+ 'monuments': {
+ '2016': {
'count': 26,
'usage': 0,
'usercount': 2,
'userreg': 2
}
}
},
- u'Turkey': {
- u'earth': {
- u'2015': {
+ 'Turkey': {
+ 'earth': {
+ '2015': {
'count': 5,
'usage': 0,
'usercount': 1,
'userreg': 0
}
},
- u'monuments': {
- u'2016': {
+ 'monuments': {
+ '2016': {
'count': 5,
'usage': 0,
'usercount': 1,
'userreg': 0
}
}
}
}
self.assertEqual(result, expected)
def test_get_events_data(self):
result = functions.get_events_data(self.data)
expected = {
- u'africa': {
- u'2014': {
+ 'africa': {
+ '2014': {
'count': 5,
'country_count': 1,
'usage': 0,
'usercount': 1,
'userreg': 0
}
},
- u'earth': {
- u'2015': {
+ 'earth': {
+ '2015': {
'count': 31,
'country_count': 2,
'usage': 0,
'usercount': 3,
'userreg': 2
}
},
- u'food': {
- u'2017': {
+ 'food': {
+ '2017': {
'count': 9,
'country_count': 1,
'usage': 0,
'usercount': 1,
'userreg': 0
}
},
- u'folklore': {
- u'2022': {
+ 'folklore': {
+ '2022': {
'count': 9,
'country_count': 1,
'usage': 0,
'usercount': 1,
'userreg': 0
}
},
- u'monuments': {
- u'2016': {
+ 'monuments': {
+ '2016': {
'count': 31,
'country_count': 2,
'usage': 0,
'usercount': 3,
'userreg': 2
}
},
- u'public_art': {
- u'2013': {
+ 'public_art': {
+ '2013': {
'count': 5,
'country_count': 1,
'usage': 0,
'usercount': 1,
'userreg': 0
}
},
- u'science': {
- u'2017': {
+ 'science': {
+ '2017': {
'count': 9,
'country_count': 1,
'usage': 0,
'usercount': 1,
'userreg': 0
}
}
}
self.assertEqual(result, expected)
def test_get_menu(self):
result = functions.get_menu(self.data)
expected = {
- u'earth': [u'2015'],
- u'monuments': [u'2016'],
- u'africa': [u'2014'],
- u'public_art': [u'2013'],
- u'science': [u'2017'],
- u'food': [u'2017'],
- u'folklore': [u'2022']
+ 'earth': ['2015'],
+ 'monuments': ['2016'],
+ 'africa': ['2014'],
+ 'public_art': ['2013'],
+ 'science': ['2017'],
+ 'food': ['2017'],
+ 'folklore': ['2022']
}
self.assertEqual(result, expected)
def test_get_edition_data(self):
result = functions.get_edition_data(self.data, 'monuments2016')
expected = {
- u'Turkey': {
- u'count': 5,
- u'category': u'Images_from_Wiki_Loves_Monuments_2016_in_Turkey',
- u'end': 20160930205959,
- u'start': 20160831210000,
- u'userreg': 0,
- u'usage': 0,
- u'data': {
- u'20160903': {
+ 'Turkey': {
+ 'count': 5,
+ 'category': 'Images_from_Wiki_Loves_Monuments_2016_in_Turkey',
+ 'end': 20160930205959,
+ 'start': 20160831210000,
+ 'userreg': 0,
+ 'usage': 0,
+ 'data': {
+ '20160903': {
"images": 5,
"joiners": 1,
"newbie_joiners": 0,
}
},
- u'usercount': 1,
+ 'usercount': 1,
},
- u'Panama': {
- u'count': 26,
- u'category': u'Images_from_Wiki_Loves_Monuments_2016_in_Panama',
- u'end': 20161001045959,
- u'start': 20160901050000,
- u'userreg': 2,
- u'usage': 0,
- u'data': {
- u'20160902': {
- u'images': 4,
- u'joiners': 1,
- u'newbie_joiners': 1,
+ 'Panama': {
+ 'count': 26,
+ 'category': 'Images_from_Wiki_Loves_Monuments_2016_in_Panama',
+ 'end': 20161001045959,
+ 'start': 20160901050000,
+ 'userreg': 2,
+ 'usage': 0,
+ 'data': {
+ '20160902': {
+ 'images': 4,
+ 'joiners': 1,
+ 'newbie_joiners': 1,
},
- u'20160903': {
- u'images': 22,
- u'joiners': 1,
- u'newbie_joiners': 1,
+ '20160903': {
+ 'images': 22,
+ 'joiners': 1,
+ 'newbie_joiners': 1,
}
},
- u'usercount': 2,
+ 'usercount': 2,
}
}
self.assertEqual(result, expected)
def test_get_instance_users_data(self):
result = functions.get_instance_users_data(self.data, 'monuments2016', 'Panama')
expected = [
- (u'Edwin Bermudez', {u'reg': 20160903173639, u'usage': 0, u'count': 22}),
- (u'Jonas David', {u'reg': 20160902064618, u'usage': 0, u'count': 4})
+ ('Edwin Bermudez', {'reg': 20160903173639, 'usage': 0, 'count': 22}),
+ ('Jonas David', {'reg': 20160902064618, 'usage': 0, 'count': 4})
]
self.assertEqual(result, expected)
if __name__ == "__main__":
unittest.main()
diff --git a/tox.ini b/tox.ini
index 5f78f96..10f6e3c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,46 +1,46 @@
[tox]
envlist =
- bashate,isort,jsonlint,flake8,py27,ansible-lint
+ bashate,isort,jsonlint,flake8,py37,ansible-lint
skipsdist = True
[testenv]
-basepython = python2.7
+basepython = python3.7
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/requirements-dev.txt
commands = nosetests {posargs}
whitelist_externals = bash
[testenv:bashate]
deps = bashate==0.5.1
commands =
bash -c "grep --recursive --binary-files=without-match \
--files-with-match '^.!.*\(ba\)\?sh$' \
--exclude-dir .tox \
--exclude-dir .git \
{toxinidir} | xargs bashate --error . --verbose"
[testenv:isort]
deps = isort==4.2.15
commands =
bash -c "find {toxinidir} \
-type d \
\( \
-path {toxinidir}/.git -o \
-path {toxinidir}/.tox -o \
-path {toxinidir}/.venv \
\) -prune -o \
-name '*.py' \
-print | xargs isort {posargs:--check-only} --verbose"
[testenv:jsonlint]
commands =
bash -c "set -euo pipefail && find conf/ -type f -name '*.json' | xargs -t -n1 python -m json.tool 2>&1 > /dev/null"
[testenv:flake8]
deps = flake8
commands = flake8
[testenv:ansible-lint]
deps = ansible-lint
commands = ansible-lint deploy/main.yml