It looks like ores started returning 500s since this morning at 6:25, I'm guessing from the logs following a restart/reload of workers, e.g. ores1006 below. Also relevant dashboard https://grafana.wikimedia.org/dashboard/db/ores?orgId=1&from=1543470112860&to=1543475918563
2018-11-29 06:25:35,337 WARNING revscoring.scoring.environment: Differences between the current environment and the environment in which the model was constructed environment were detected: - python_build ('default', 'Jan 19 2017 14:11:04') mismatch with original environment ('default', 'Sep 27 2018 17:25:39') - python_compiler 'GCC 6.3.0 20170118' mismatch with original environment 'GCC 6.3.0 20170516' - release '4.9.0-6-amd64' mismatch with original environment '4.9.0-8-amd64' - revscoring_version '2.2.5' mismatch with original environment '2.2.6' - version '#1 SMP Debian 4.9.82-1+deb9u3 (2018-03-02)' mismatch with original environment '#1 SMP Debian 4.9.110-3+deb9u4 (2018-08-21)' - platform 'Linux-4.9.0-6-amd64-x86_64-with-debian-9.4' mismatch with original environment 'Linux-4.9.0-8-amd64-x86_64-with-debian-9.5' 2018-11-29 06:25:35,721 ERROR ores.wsgi.responses: Traceback (most recent call last): File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/serialization.py", line 50, in _reraise_errors yield File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/serialization.py", line 221, in dumps payload = encoder(data) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/utils/json.py", line 69, in dumps **dict(default_kwargs, **kwargs)) File "/usr/lib/python3.5/json/__init__.py", line 237, in dumps **kw).encode(obj) File "/usr/lib/python3.5/json/encoder.py", line 198, in encode chunks = self.iterencode(o, _one_shot=True) File "/usr/lib/python3.5/json/encoder.py", line 256, in iterencode return _iterencode(o, 0) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/utils/json.py", line 59, in default return super(JSONEncoder, self).default(o) File "/usr/lib/python3.5/json/encoder.py", line 179, in default raise TypeError(repr(o) + " is not JSON serializable") TypeError: frozenset({'itemquality'}) is not JSON serializable During handling of the above exception, another exception occurred: Traceback (most recent call last): File "./ores/wsgi/routes/v3/util.py", line 101, in process_score_request score_response = scoring_system.score(score_request) File "./ores/scoring_systems/scoring_system.py", line 59, in score response = self._score(request) File "./ores/scoring_systems/celery_queue.py", line 195, in _score return super()._score(*args, **kwargs) File "./ores/scoring_systems/scoring_system.py", line 118, in _score inprogress_results=inprogress_results) File "./ores/scoring_systems/celery_queue.py", line 100, in _process_missing_scores request.to_json(), missing_models, rev_id, root_cache) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/celery/app/task.py", line 412, in delay return self.apply_async(args, kwargs) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/celery/app/task.py", line 535, in apply_async **options File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/celery/app/base.py", line 728, in send_task amqp.send_task_message(P, name, message, **options) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/celery/app/amqp.py", line 552, in send_task_message **properties File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/messaging.py", line 169, in publish compression, headers) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/messaging.py", line 252, in _prepare body) = dumps(body, serializer=serializer) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/serialization.py", line 221, in dumps payload = encoder(data) File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__ self.gen.throw(type, value, traceback) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/serialization.py", line 54, in _reraise_errors reraise(wrapper, wrapper(exc), sys.exc_info()[2]) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/vine/five.py", line 178, in reraise raise value.with_traceback(tb) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/serialization.py", line 50, in _reraise_errors yield File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/serialization.py", line 221, in dumps payload = encoder(data) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/utils/json.py", line 69, in dumps **dict(default_kwargs, **kwargs)) File "/usr/lib/python3.5/json/__init__.py", line 237, in dumps **kw).encode(obj) File "/usr/lib/python3.5/json/encoder.py", line 198, in encode chunks = self.iterencode(o, _one_shot=True) File "/usr/lib/python3.5/json/encoder.py", line 256, in iterencode return _iterencode(o, 0) File "/srv/deployment/ores/deploy/venv/lib/python3.5/site-packages/kombu/utils/json.py", line 59, in default return super(JSONEncoder, self).default(o) File "/usr/lib/python3.5/json/encoder.py", line 179, in default raise TypeError(repr(o) + " is not JSON serializable") kombu.exceptions.EncodeError: frozenset({'itemquality'}) is not JSON serializable