/usr/local/bin/mwscript extensions/Wikibase/repo/maintenance/dispatchChanges.php --wiki wikidatawiki

root@mwmaint2001:~# crontab -l -uwww-data | grep -w wikidatawiki
*/3 * * * * echo "$$: Starting dispatcher" >> /var/log/wikidata/dispatchChanges-wikidatawiki.log; /usr/local/bin/mwscript extensions/Wikibase/repo/maintenance/dispatchChanges.php --wiki wikidatawiki >> /var/log/wikidata/dispatchChanges-wikidatawiki.log 2>&1; echo "$$: Dispatcher exited with $?" >> /var/log/wikidata/dispatchChanges-wikidatawiki.log

SELECT /* Wikibase\Lib\Store\Sql\Terms\DatabaseTermInLangIdsResolver::selectTermsViaJoin xx */ wbtl_id, wbtl_type_id, wbxl_language, wbx_text, wbpt_property_id FROM `wbt_term_in_lang` JOIN `wbt_text_in_lang` ON ((wbtl_text_in_lang_id=wbxl_id)) JOIN `wbt_text` ON ((wbxl_text_id=wbx_id)) JOIN `wbt_property_terms` ON ((`wbpt_term_in_lang_id` = wbtl_id)) WHERE wbtl_type_id = 1 AND wbxl_language = 'lez'

root@db2081.codfw.wmnet[wikidatawiki]> FLUSH STATUS; pager cat > /dev/null;
Query OK, 0 rows affected (0.032 sec)

PAGER set to 'cat > /dev/null'
root@db2081.codfw.wmnet[wikidatawiki]> SELECT /* Wikibase\Lib\Store\Sql\Terms\DatabaseTermInLangIdsResolver::selectTermsViaJoin 4 */ wbtl_id, wbtl_type_id, wbxl_language, wbx_text, wbpt_property_id FROM `wbt_term_in_lang` JOIN `wbt_text_in_lang` ON ((wbtl_text_in_lang_id=wbxl_id)) JOIN `wbt_text` ON ((wbxl_text_id=wbx_id)) JOIN `wbt_property_terms` ON ((`wbpt_term_in_lang_id` = wbtl_id)) WHERE wbtl_type_id = 1 AND wbxl_language = 'lez' ; nopager;
14 rows in set (1.781 sec)

PAGER set to stdout
root@db2081.codfw.wmnet[wikidatawiki]> SHOW STATUS like 'Hand%';
+----------------------------+--------+
| Variable_name              | Value  |
+----------------------------+--------+
| Handler_commit             | 1      |
| Handler_delete             | 0      |
| Handler_discover           | 0      |
| Handler_external_lock      | 0      |
| Handler_icp_attempts       | 0      |
| Handler_icp_match          | 0      |
| Handler_mrr_init           | 0      |
| Handler_mrr_key_refills    | 0      |
| Handler_mrr_rowid_refills  | 0      |
| Handler_prepare            | 0      |
| Handler_read_first         | 1      |
| Handler_read_key           | 531005 |
| Handler_read_last          | 0      |
| Handler_read_next          | 364794 |
| Handler_read_prev          | 0      |
| Handler_read_retry         | 0      |
| Handler_read_rnd           | 0      |
| Handler_read_rnd_deleted   | 0      |
| Handler_read_rnd_next      | 0      |
| Handler_rollback           | 0      |
| Handler_savepoint          | 0      |
| Handler_savepoint_rollback | 0      |
| Handler_tmp_delete         | 0      |
| Handler_tmp_update         | 0      |
| Handler_tmp_write          | 0      |
| Handler_update             | 0      |
| Handler_write              | 0      |
+----------------------------+--------+
27 rows in set (0.033 sec)

1:23 PM <addshore> I also notice this? https://grafana.wikimedia.org/d/000000202/api-frontend-summary?orgId=1&from=1602020402068&to=1602023755950 REST API backend internla requests spike?
1:23 PM <addshore> also happened a week ago? https://grafana.wikimedia.org/d/000000202/api-frontend-summary?orgId=1&from=1601674802000&to=1601678155000
1:23 PM <addshore> zooming out I see even more spikes https://grafana.wikimedia.org/d/000000202/api-frontend-summary?orgId=1&from=now-7d&to=now
1:24 PM <addshore> I have 0 idea what this is though or thus if it may or may not be related
1:24 PM <marostegui> addshore: looks like it has been happening since 1st oct yeah
1:24 PM <marostegui> per the graphs reported
1:24 PM <addshore> but that is a regulrly occouring spike at the right time 
1:24 PM <addshore> I'll write that and those links in the ticket?
1:25 PM <marostegui> addshore: yeah, daily at 22 UTC, which is weeeeeeird
1:26 PM <addshore> yeah, also a correlation with RED dashboard issues on the 3rd https://grafana.wikimedia.org/d/RIA1lzDZk/application-servers-red-dashboard?orgId=1&from=1601760432456&to=1601764464836

1:57 PM <addshore> While i was browsing around I also saw spikes in action=query in grafana, but couldn't dive much deeper, but I guess wikifeeds calls that, and some of the wikibase api modules there Amir1
...
1:58 PM <addshore> which I guess is https://en.wikipedia.org/w/api.php?action=help&recursivesubmodules=1#query+pageterms Amir1 
1:58 PM <Amir1> hmm addshore maybe we can coordinate with them to improve their API calls or build something suited to their needs?
...
1:58 PM <Amir1> addshore: aaah, we don't cache pageterms
1:58 PM <addshore> Amir1: yeah, im not sure how much caching pageterms currently has?
1:59 PM <Amir1> I wanted to do it, you didn't let me 
1:59 PM <Amir1> :D
1:59 PM <addshore> Amir1: there you go, so thats why the spikes end up getting all the way down to s8? :P
1:59 PM <addshore> Amir1: well, now we have a reason to :D