We are seeing lots of errors related with the following output (https://logstash.wikimedia.org/goto/f8ec4b3e22a8b3d5a8ad0143a4f8982f):
{ "_index": "logstash-2018.08.22", "_type": "mediawiki", "_id": "AWVgIepq8VFZOIjHYi4M", "_version": 1, "_score": null, "_source": { "server": "www.mediawiki.org", "unique_id": "W3z2PwpAAEUAADLO8AUAAAAR", "db_server": "10.64.32.136", "method": "Wikimedia\\Rdbms\\DatabaseMysqlBase::open", "level": "ERROR", "ip": "10.64.48.103", "wiki": "mediawikiwiki", "channel": "DBConnection", "mwversion": "1.32.0-wmf.18", "message": "Error connecting to 10.64.32.136: Unknown database 'wikidatawiki'", "type": "mediawiki", "error": "Unknown database 'wikidatawiki'", "normalized_message": "Error connecting to {db_server}: {error}", "url": "/wiki/Special:MyLanguage/Help:Logging_in", "tags": [ "syslog", "es", "es" ], "reqId": "W3z2PwpAAEUAADLO8AUAAAAR", "referrer": "http://hydrodictyon.eeb.uconn.edu/eebedia/index.php?title=Special:UserLogin&returnto=Main+Page", "@timestamp": "2018-08-22T05:36:00.000Z", "http_method": "GET", "db_name": "wikidatawiki", "db_user": "wikiuser", "@version": 1, "host": "mw1274", "shard": "s3" }, "fields": { "@timestamp": [ 1534916160000 ] }, "sort": [ 1534916160000 ] }
The interesting bits are:
server:www.mediawiki.org method:Wikimedia\Rdbms\DatabaseMysqlBase::open wiki:mediawikiwiki message:Error connecting to 10.64.48.35: Unknown database 'wikidatawiki' type:mediawiki error:Unknown database 'wikidatawiki' shard:s3
wikidatawiki isn't on s3, but on s8.
Everything is coming from www.mediawiki.org apparently, and it is indeed causing an error:
https://www.mediawiki.org/wiki/Help:Logging_in returns:
2018-08-22 05:35:54 [W3z2OgpAIDAAAKUJ1hIAAABY] mw1327 mediawikiwiki 1.32.0-wmf.18 exception ERROR: [W3z2OgpAIDAAAKUJ1hIAAABY] /wiki/Help:Logging_in Wikibase\DataModel\Services\Lookup\EntityLookupException from line 44 of /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/RevisionBasedEntityLookup.php: Cannot access the database: Unknown error (10.64.48.35) {"exception_id":"W3z2OgpAIDAAAKUJ1hIAAABY","exception_url":"/wiki/Help:Logging_in","caught_by":"mwe_handler"} [Exception Wikibase\DataModel\Services\Lookup\EntityLookupException] (/srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/RevisionBasedEntityLookup.php:44) Cannot access the database: Unknown error (10.64.48.35) #0 /srv/mediawiki/php-1.32.0-wmf.18/vendor/wikibase/data-model-services/src/Lookup/RedirectResolvingEntityLookup.php(51): Wikibase\Lib\Store\RevisionBasedEntityLookup->getEntity(Wikibase\DataModel\Entity\ItemId) #1 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/LangLinkHandler.php(109): Wikibase\DataModel\Services\Lookup\RedirectResolvingEntityLookup->getEntity(Wikibase\DataModel\Entity\ItemId) #2 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/LangLinkHandler.php(331): Wikibase\Client\LangLinkHandler->getEntityLinks(Title) #3 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/LangLinkHandler.php(352): Wikibase\Client\LangLinkHandler->getEffectiveRepoLinks(Title, ParserOutput) #4 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/Hooks/ParserOutputUpdateHookHandlers.php(97): Wikibase\Client\LangLinkHandler->addLinksFromRepository(Title, ParserOutput) #5 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/Hooks/ParserOutputUpdateHookHandlers.php(65): Wikibase\Client\Hooks\ParserOutputUpdateHookHandlers->doContentAlterParserOutput(Title, ParserOutput) #6 /srv/mediawiki/php-1.32.0-wmf.18/includes/Hooks.php(174): Wikibase\Client\Hooks\ParserOutputUpdateHookHandlers::onContentAlterParserOutput(WikitextContent, Title, ParserOutput) #7 /srv/mediawiki/php-1.32.0-wmf.18/includes/Hooks.php(202): Hooks::callHook(string, array, array, NULL) #8 /srv/mediawiki/php-1.32.0-wmf.18/includes/content/AbstractContent.php(520): Hooks::run(string, array) #9 /srv/mediawiki/php-1.32.0-wmf.18/includes/poolcounter/PoolWorkArticleView.php(145): AbstractContent->getParserOutput(Title, integer, ParserOptions) #10 /srv/mediawiki/php-1.32.0-wmf.18/includes/poolcounter/PoolCounterWork.php(123): PoolWorkArticleView->doWork() #11 /srv/mediawiki/php-1.32.0-wmf.18/includes/page/Article.php(604): PoolCounterWork->execute() #12 /srv/mediawiki/php-1.32.0-wmf.18/includes/actions/ViewAction.php(68): Article->view() #13 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(501): ViewAction->show() #14 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(294): MediaWiki->performAction(Article, Title) #15 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(868): MediaWiki->performRequest() #16 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(525): MediaWiki->main() #17 /srv/mediawiki/php-1.32.0-wmf.18/index.php(42): MediaWiki->run() #18 /srv/mediawiki/w/index.php(3): include(string) #19 {main} Caused by: [Exception Wikimedia\Rdbms\DBConnectionError] (/srv/mediawiki/php-1.32.0-wmf.18/includes/libs/rdbms/loadbalancer/LoadBalancer.php:1179) Cannot access the database: Unknown error (10.64.48.35) #0 /srv/mediawiki/php-1.32.0-wmf.18/includes/libs/rdbms/loadbalancer/LoadBalancer.php(768): Wikimedia\Rdbms\LoadBalancer->reportConnectionError() #1 /srv/mediawiki/php-1.32.0-wmf.18/includes/Storage/SqlBlobStore.php(203): Wikimedia\Rdbms\LoadBalancer->getConnection(integer, array, string) #2 /srv/mediawiki/php-1.32.0-wmf.18/includes/Storage/SqlBlobStore.php(279): MediaWiki\Storage\SqlBlobStore->getDBConnection(integer) #3 /srv/mediawiki/php-1.32.0-wmf.18/includes/libs/objectcache/WANObjectCache.php(1246): Closure$MediaWiki\Storage\SqlBlobStore::getBlob(boolean, integer, array, NULL) #4 /srv/mediawiki/php-1.32.0-wmf.18/includes/libs/objectcache/WANObjectCache.php(1119): WANObjectCache->doGetWithSetCallback(string, integer, Closure$MediaWiki\Storage\SqlBlobStore::getBlob;2741, array) #5 /srv/mediawiki/php-1.32.0-wmf.18/includes/Storage/SqlBlobStore.php(284): WANObjectCache->getWithSetCallback(string, integer, Closure$MediaWiki\Storage\SqlBlobStore::getBlob;2741, array) #6 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/Sql/WikiPageEntityRevisionLookup.php(206): MediaWiki\Storage\SqlBlobStore->getBlob(string) #7 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/Sql/WikiPageEntityRevisionLookup.php(114): Wikibase\Lib\Store\Sql\WikiPageEntityRevisionLookup->loadEntity(stdClass) #8 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/DispatchingEntityRevisionLookup.php(59): Wikibase\Lib\Store\Sql\WikiPageEntityRevisionLookup->getEntityRevision(Wikibase\DataModel\Entity\ItemId, integer, string) #9 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/CachingEntityRevisionLookup.php(104): Wikibase\Lib\Store\DispatchingEntityRevisionLookup->getEntityRevision(Wikibase\DataModel\Entity\ItemId, integer, string) #10 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/CachingEntityRevisionLookup.php(87): Wikibase\Lib\Store\CachingEntityRevisionLookup->fetchEntityRevision(Wikibase\DataModel\Entity\ItemId, integer, string) #11 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/CachingEntityRevisionLookup.php(104): Wikibase\Lib\Store\CachingEntityRevisionLookup->getEntityRevision(Wikibase\DataModel\Entity\ItemId, integer, string) #12 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/CachingEntityRevisionLookup.php(87): Wikibase\Lib\Store\CachingEntityRevisionLookup->fetchEntityRevision(Wikibase\DataModel\Entity\ItemId, integer, string) #13 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/lib/includes/Store/RevisionBasedEntityLookup.php(39): Wikibase\Lib\Store\CachingEntityRevisionLookup->getEntityRevision(Wikibase\DataModel\Entity\ItemId) #14 /srv/mediawiki/php-1.32.0-wmf.18/vendor/wikibase/data-model-services/src/Lookup/RedirectResolvingEntityLookup.php(51): Wikibase\Lib\Store\RevisionBasedEntityLookup->getEntity(Wikibase\DataModel\Entity\ItemId) #15 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/LangLinkHandler.php(109): Wikibase\DataModel\Services\Lookup\RedirectResolvingEntityLookup->getEntity(Wikibase\DataModel\Entity\ItemId) #16 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/LangLinkHandler.php(331): Wikibase\Client\LangLinkHandler->getEntityLinks(Title) #17 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/LangLinkHandler.php(352): Wikibase\Client\LangLinkHandler->getEffectiveRepoLinks(Title, ParserOutput) #18 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/Hooks/ParserOutputUpdateHookHandlers.php(97): Wikibase\Client\LangLinkHandler->addLinksFromRepository(Title, ParserOutput) #19 /srv/mediawiki/php-1.32.0-wmf.18/extensions/Wikibase/client/includes/Hooks/ParserOutputUpdateHookHandlers.php(65): Wikibase\Client\Hooks\ParserOutputUpdateHookHandlers->doContentAlterParserOutput(Title, ParserOutput) #20 /srv/mediawiki/php-1.32.0-wmf.18/includes/Hooks.php(174): Wikibase\Client\Hooks\ParserOutputUpdateHookHandlers::onContentAlterParserOutput(WikitextContent, Title, ParserOutput) #21 /srv/mediawiki/php-1.32.0-wmf.18/includes/Hooks.php(202): Hooks::callHook(string, array, array, NULL) #22 /srv/mediawiki/php-1.32.0-wmf.18/includes/content/AbstractContent.php(520): Hooks::run(string, array) #23 /srv/mediawiki/php-1.32.0-wmf.18/includes/poolcounter/PoolWorkArticleView.php(145): AbstractContent->getParserOutput(Title, integer, ParserOptions) #24 /srv/mediawiki/php-1.32.0-wmf.18/includes/poolcounter/PoolCounterWork.php(123): PoolWorkArticleView->doWork() #25 /srv/mediawiki/php-1.32.0-wmf.18/includes/page/Article.php(604): PoolCounterWork->execute() #26 /srv/mediawiki/php-1.32.0-wmf.18/includes/actions/ViewAction.php(68): Article->view() #27 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(501): ViewAction->show() #28 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(294): MediaWiki->performAction(Article, Title) #29 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(868): MediaWiki->performRequest() #30 /srv/mediawiki/php-1.32.0-wmf.18/includes/MediaWiki.php(525): MediaWiki->main() #31 /srv/mediawiki/php-1.32.0-wmf.18/index.php(42): MediaWiki->run() #32 /srv/mediawiki/w/index.php(3): include(string) #33 {main}