diff --git a/Pipfile b/Pipfile index c82cdd5..1383eb9 100644 --- a/Pipfile +++ b/Pipfile @@ -1,16 +1,18 @@ [[source]] url = "https://pypi.org/simple" verify_ssl = true name = "pypi" [requires] python_version = "3.7" [dev-packages] pytest = "*" pytest-cov = "*" +fakeredis = "*" [packages] pywikibot = "*" ww = "*" flask = "*" +redis = "*" diff --git a/Pipfile.lock b/Pipfile.lock index a149e54..e67feab 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,307 +1,338 @@ { "_meta": { "hash": { - "sha256": "3cd7f313b7f38346f982482c9512a20581ea01ae1b07c7c535deec6fc936cf31" + "sha256": "616f66331d5453624d79385192fd587d68860fe45982001b977a77d566d20b8e" }, "pipfile-spec": 6, "requires": { "python_version": "3.7" }, "sources": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true } ] }, "default": { "certifi": { "hashes": [ "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" ], "version": "==2020.4.5.1" }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], "version": "==3.0.4" }, "click": { "hashes": [ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==7.1.2" }, "flask": { "hashes": [ "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" ], "index": "pypi", "version": "==1.1.2" }, "formatizer": { "hashes": [ "sha256:3061ced1daa08f1836b79f4a3de16a33a54179331273e0b9c757d27ab339c29f" ], "version": "==0.1.1" }, "future": { "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.2" }, "idna": { "hashes": [ "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9" }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "jinja2": { "hashes": [ "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.11.2" }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" }, "pywikibot": { "hashes": [ "sha256:c61095a8d56c02ebc73f5c23f7e75248f11e0272d88cbe6b5d258b0438cbb0b6" ], "index": "pypi", "version": "==3.0.20200508" }, + "redis": { + "hashes": [ + "sha256:6e9d2722a95d10ddf854596e66516d316d99c6a483e5db3b35c34e1158b2bfa1", + "sha256:a5b0e25890d216d8189636742c50ab992e42eea699bcc1b08cc2d6bf3adff52a" + ], + "index": "pypi", + "version": "==3.5.1" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.23.0" }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.14.0" }, "urllib3": { "hashes": [ "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", "version": "==1.25.9" }, "werkzeug": { "hashes": [ "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.1" }, "ww": { "hashes": [ "sha256:02d2b9ea134317901c889fc844958630f478b5d1a98d5938ce787cf92d02b8ed", "sha256:3664f1f91bf927fe597ab153e8df73c8954927258b3737220efd1cb9912ebd7e" ], "index": "pypi", "version": "==0.2.1" } }, "develop": { "attrs": { "hashes": [ "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==19.3.0" }, "coverage": { "hashes": [ "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", "version": "==5.1" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "importlib-metadata": { "hashes": [ "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" ], "markers": "python_version < '3.8'", "version": "==1.6.0" }, "more-itertools": { "hashes": [ "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" ], "markers": "python_version >= '3.5'", "version": "==8.2.0" }, "packaging": { "hashes": [ "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.3" }, "pluggy": { "hashes": [ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.13.1" }, "py": { "hashes": [ "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.8.1" }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytest": { "hashes": [ "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3", "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698" ], "index": "pypi", "version": "==5.4.2" }, "pytest-cov": { "hashes": [ "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b", "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626" ], "index": "pypi", "version": "==2.8.1" }, + "redis": { + "hashes": [ + "sha256:6e9d2722a95d10ddf854596e66516d316d99c6a483e5db3b35c34e1158b2bfa1", + "sha256:a5b0e25890d216d8189636742c50ab992e42eea699bcc1b08cc2d6bf3adff52a" + ], + "index": "pypi", + "version": "==3.5.1" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.14.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "wcwidth": { "hashes": [ "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1", "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1" ], "version": "==0.1.9" }, "zipp": { "hashes": [ "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" ], "markers": "python_version >= '3.6'", "version": "==3.1.0" } } } diff --git a/docker-compose.yml b/docker-compose.yml index b013bec..d1657b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,22 @@ version: '2' services: web: build: context: . dockerfile: conf/Dockerfile.web command: flask run --host=0.0.0.0 environment: - FLASK_APP=integraality/app.py - FLASK_ENV=development - PYWIKIBOT_NO_USER_CONFIG=1 + - REDIS_HOST=redis - PYTHONPATH=/code/integraality/ ports: - 5000:5000 volumes: - .:/code + + redis: + image: redis:3.2.6 + ports: + - 6379:6379 diff --git a/integraality/cache.py b/integraality/cache.py new file mode 100644 index 0000000..c087b57 --- /dev/null +++ b/integraality/cache.py @@ -0,0 +1,46 @@ +# Inspired/copied from https://github.com/taylorhakes/python-redis-cache/blob/master/redis_cache/__init__.py, MIT-licensed + +import pickle + +DEFAULT_TTL = 604800 # 1 week + + +class RedisCache: + + def __init__(self, cache_client, prefix="integraality"): + self.prefix = prefix + self.client = cache_client + + def make_key(self, key): + return "{0}:{1}".format(self.prefix, key) + + def get_cache_value(self, key): + ns_key = self.make_key(key) + cached_value = self.client.get(ns_key) + if cached_value: + return pickle.loads(cached_value) + else: + return None + + def set_cache_value(self, key, value): + ns_key = self.make_key(key) + cached_value = pickle.dumps(value) + pipe = self.client.pipeline() + pipe.set(ns_key, cached_value) + pipe.expire(ns_key, DEFAULT_TTL) + pipe.execute() + + def invalidate(self, key): + ns_key = self.make_key(key) + print("Invalidating key %s" % ns_key) + pipe = self.client.pipeline() + pipe.delete(ns_key) + pipe.execute() + + def list_keys(self): + keys = self.client.keys() + print("%s keys in Redis" % len(keys)) + + def flushall(self): + print("Deleting all keys") + self.client.flushall() diff --git a/integraality/pages_processor.py b/integraality/pages_processor.py index e19397b..239ee04 100644 --- a/integraality/pages_processor.py +++ b/integraality/pages_processor.py @@ -1,189 +1,215 @@ #!/usr/bin/python # -*- coding: utf-8 -*- """ Bot to generate statistics """ +import os import re +from redis import StrictRedis from ww import f import pywikibot from pywikibot import pagegenerators +from cache import RedisCache from property_statistics import ( DescriptionConfig, LabelConfig, PropertyConfig, PropertyStatistics, QueryException ) REQUIRED_CONFIG_FIELDS = ['selector_sparql', 'grouping_property', 'properties'] class ProcessingException(Exception): pass class ConfigException(ProcessingException): pass class NoEndTemplateException(ProcessingException): pass class PagesProcessor: - def __init__(self, url="https://www.wikidata.org/wiki/"): + def __init__(self, url="https://www.wikidata.org/wiki/", cache_client=None): self.site = pywikibot.Site(url=url) self.template_name = 'Property dashboard' self.end_template_name = 'Property dashboard end' self.summary = u'Update property usage stats' self.outputs = [] + if not cache_client: + host = os.getenv("REDIS_HOST", 'tools-redis.svc.eqiad.wmflabs') + cache_client = StrictRedis(host=host, decode_responses=False) + self.cache = RedisCache(cache_client=cache_client) + + def make_cache_key(self, page_title): + return ":".join([self.site.code, page_title]).replace(" ", "_") + def get_all_pages(self): template = pywikibot.Page(self.site, self.template_name, ns=10) return pagegenerators.ReferringPageGenerator(template, onlyTemplateInclusion=True) @staticmethod def extract_elements_from_template_param(template_param): """Extract and sanitize the contents of a parsed template param.""" (field, _, value) = template_param.partition(u'=') return (field.strip(), value.replace('{{!}}', '|')) def parse_config_from_params(self, params): return { key: value for (key, value) in [self.extract_elements_from_template_param(param) for param in params] if key } - def make_stats_object_for_page(self, page): + def make_stats_object_arguments_for_page(self, page): all_templates_with_params = page.templatesWithParams() if self.end_template_name not in [template.title(with_ns=False) for (template, _) in all_templates_with_params]: raise NoEndTemplateException("No end template '%s' provided" % self.end_template_name) start_templates_with_params = [ (template, params) for (template, params) in all_templates_with_params if template.title(with_ns=False) == self.template_name ] if not start_templates_with_params: msg = ( "No start template '%s' found, which is an impossible situation. " "This is potentially an upstream pywikibot issue." % self.template_name ) raise ConfigException(msg) if len(start_templates_with_params) > 1: pywikibot.warn("More than one template on the page %s" % page.title()) (template, params) = start_templates_with_params[0] parsed_config = self.parse_config_from_params(params) config = self.parse_config(parsed_config) + key = self.make_cache_key(page.title()) + self.cache.set_cache_value(key, config) + return config + + def make_stats_object_for_page(self, page): + config = self.make_stats_object_arguments_for_page(page) try: return PropertyStatistics(**config) except TypeError: raise ConfigException("The template parameters are incorrect.") def process_page(self, page): + self.cache.invalidate(self.make_cache_key(page.title())) stats = self.make_stats_object_for_page(page) try: output = stats.retrieve_and_process_data() except QueryException as e: raise ConfigException(e) new_text = self.replace_in_page(output, page.get()) page.put(new_text, self.summary) def parse_config(self, config): for field in REQUIRED_CONFIG_FIELDS: if field not in config: pywikibot.output("Missing required field %s" % field) raise ConfigException("A required field is missing: %s" % field) config['columns'] = self.parse_config_properties(config['properties']) del config['properties'] config['stats_for_no_group'] = bool(config.get('stats_for_no_group', False)) return config @staticmethod def parse_config_properties(properties_string): properties = properties_string.split(',') properties_data = [] for prop in properties: try: (key, title) = prop.split(':') except ValueError: (key, title) = (prop, None) if key: if key.startswith('P'): splitted = key.split('/') if len(splitted) == 3: (property_name, value, qualifier) = splitted elif len(splitted) == 2: (property_name, value, qualifier) = (splitted[0], None, splitted[1]) else: (property_name, value, qualifier) = (key, None, None) entry = PropertyConfig(property=property_name, title=title, qualifier=qualifier, value=value) elif key.startswith('L'): entry = LabelConfig(language=key[1:]) elif key.startswith('D'): entry = DescriptionConfig(language=key[1:]) properties_data.append(entry) return properties_data def replace_in_page(self, output, page_text): regex_text = f('({{{{{self.template_name}.*?(?= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' coverage==5.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4' +fakeredis==1.4.1 importlib-metadata==1.6.0; python_version < '3.8' more-itertools==8.2.0; python_version >= '3.5' packaging==20.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' py==1.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' pytest-cov==2.8.1 pytest==5.4.2 +redis==3.5.1 six==1.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +sortedcontainers==2.1.0 wcwidth==0.1.9 zipp==3.1.0; python_version >= '3.6' diff --git a/requirements.txt b/requirements.txt index a514be1..b5355ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,18 @@ -i https://pypi.org/simple certifi==2020.4.5.1 chardet==3.0.4 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' flask==1.1.2 formatizer==0.1.1 future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' idna==2.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' itsdangerous==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' jinja2==2.11.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pywikibot==3.0.20200508 +redis==3.5.1 requests==2.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' six==1.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' urllib3==1.25.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4' werkzeug==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' ww==0.2.1 diff --git a/setup.cfg b/setup.cfg index 66371aa..7e3be95 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,14 +1,15 @@ [flake8] max-line-length = 150 ignore = W503,F841 [isort] -known_third_party = ww,flask +known_third_party = ww,flask,redis +known_testing = fakeredis known_pywikibot = pywikibot multi_line_output = 3 -sections = FUTURE,STDLIB,THIRDPARTY,PYWIKIBOT,FIRSTPARTY,LOCALFOLDER +sections = FUTURE,STDLIB,THIRDPARTY,TESTING,PYWIKIBOT,FIRSTPARTY,LOCALFOLDER [coverage:run] source = integraality branch = True omit = **/tests/*