Page MenuHomePhabricator
Paste P4142

Convert erfgoedbot monuments_config into JSON files
ActivePublic

Authored by JeanFred on Oct 2 2016, 4:58 PM.
from collections import OrderedDict
import inspect
import io
import json
import yaml
import os
import re
import monuments_config_old
def get_config_dir():
return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'monuments_config')
def get_comments():
# ('ch', 'it'): { # Monuments in Switzerland in Italian
mapping = {}
source_code = inspect.getsourcelines(monuments_config_old)[0]
pattern = re.compile(r"""
\(
\'(.+)\'
\,\s*
\'(.+)\'
.*
\#
\s*
(.*)
""", re.X)
for line in source_code:
if '#' in line:
line = line.strip()
match = re.search(pattern, line)
if match:
(country, lang, comment) = match.groups()
mapping[(country, lang)] = comment
return mapping
class CountryConverter(object):
def __init__(self, key, data, directory, comments):
(self.country, self.lang) = key
self.data = data
self.directory = directory
self.comments = comments
def create_ordered_dictionary(self):
new_dict = OrderedDict()
fields = [
'country', 'lang', 'description',
'project', 'namespaces',
'table', 'truncate', 'primkey',
'headerTemplate', 'rowTemplate',
'commonsTemplate', 'commonsTrackerCategory',
'commonsCategoryBase', 'unusedImagesPage',
'imagesWithoutIdPage', 'missingCommonscatPage',
'registrantUrlBase', 'fields'
]
for field in fields:
value = self.data.pop(field, '')
if value or field in ['truncate']:
new_dict[field] = value
for key in self.data.keys():
if key not in ['description', 'project', 'footerTemplate', 'autoGeocode', 'countryBbox']:
print key
return new_dict
def get_filename(self):
raise NotImplementedError('get_filename must be implemented')
def get_result_string(self, result):
raise NotImplementedError('get_result_string must be implemented')
def convert(self):
filepath = os.path.join(self.directory, self.get_filename())
self.data['country'] = self.country
description = self.comments[(self.country, self.lang)]
self.data['description'] = description
result = self.create_ordered_dictionary()
with io.open(filepath, 'w', encoding='utf-8') as fp:
result_string = self.get_result_string(result)
# print result_string
fp.write(unicode(result_string))
class JsonCountryConverter(CountryConverter):
def get_filename(self):
return '%s_%s.json' % (self.country, self.lang)
def get_result_string(self, result):
return json.dumps(result, ensure_ascii=False, encoding='utf8', indent=4)
class YamlCountryConverter(CountryConverter):
def get_filename(self):
return '%s_%s.yml' % (self.country, self.lang)
def get_result_string(self, result):
return yaml.dump(result, encoding='utf-8', indent=2)
def main():
countries = monuments_config_old.countries
comments_mapping = get_comments()
converter_class = JsonCountryConverter
# converter_class = YamlCountryConverter
for key, data in countries.iteritems():
converter = converter_class(key, data, get_config_dir(), comments_mapping)
converter.convert()
if __name__ == '__main__':
main()