Page MenuHomePhabricator
Paste P8456

cdanis@grafana1001.eqiad.wmnet /var/log/apache2 % zgrep '^2019-04-25T20:3' other_vhosts_access.log.4 | sort -g -k +2 | cut -d"$(echo -ne '\t')" -f1-10 | tail -n100 | phaste
ActivePublic

Authored by ProdPasteBot on Apr 29 2019, 7:39 PM.
Tags
None
Referenced Files
F28855502: raw.txt
Apr 29 2019, 7:39 PM
Subscribers
None
2019-04-25T20:34:01 1161913 10.64.16.22 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaManager_LeaderCount%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1161941 10.64.32.67 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(kafka_server_ReplicaManager_IsrShrinks_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 1164188 10.64.16.24 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bkafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%5B5m%5D))%20by%20(topic)))&start=1548446400&end=1556229600&step=7200 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1164666 10.64.48.101 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2Ctopic%3D%22%22%7D%20%5B5m%5D))&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1168981 10.64.0.132 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaManager_PartitionCount%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1169727 10.64.16.22 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(1-%20(node_filesystem_avail_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D))%20or%20(1-%20(node_filesystem_avail%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D))&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1173482 10.64.0.130 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaManager_UnderReplicatedPartitions%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1173808 10.64.0.130 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=jvm_memory_bytes_max%7Bcluster%3D%22eventbus%22%2Carea%3D%22heap%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556229600&step=7200 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 1174187 10.64.16.24 proxy-server/503 299 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=jvm_memory_bytes_used%7Bcluster%3D%22eventbus%22%2Carea%3D%22heap%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556229600&step=7200 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1301979 10.64.48.101 proxy-server/200 28088 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(jvm_gc_collection_seconds_count%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cgc%3D%22G1%20Old%20Generation%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1303075 10.64.16.24 proxy-server/200 28013 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(kafka_server_ReplicaManager_IsrShrinks_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1314094 10.64.0.130 proxy-server/200 55133 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=increase(jvm_gc_collection_seconds_sum%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cgc%3D%22G1%20Young%20Generation%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1327604 10.64.48.103 proxy-server/200 28214 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaManager_UnderReplicatedPartitions%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1351173 10.64.32.69 proxy-server/200 28238 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_controller_KafkaController_OfflinePartitionsCount_Value%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1356262 10.64.48.103 proxy-server/200 28262 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_controller_KafkaController_PreferredReplicaImbalanceCount_Value%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1393415 10.64.16.24 proxy-server/200 28235 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_controller_KafkaController_ActiveControllerCount_Value%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1396733 10.64.48.101 proxy-server/200 84884 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=jvm_memory_bytes_max%7Bcluster%3D%22eventbus%22%2Carea%3D%22heap%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556229600&step=7200 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1514719 10.64.0.132 proxy-server/200 1907 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(kafka_server_ReplicaManager_PartitionCount%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D)&start=1548374400&end=1556236800&step=86400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1630451 10.64.16.24 proxy-server/200 31418 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaManager_LeaderCount%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1631621 10.64.32.69 proxy-server/200 31427 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaManager_PartitionCount%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1648508 10.64.0.132 proxy-server/200 28241 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_server_ReplicaFetcherManager_MaxLag%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1678874 10.64.48.101 proxy-server/200 10366 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(kafka_server_ReplicaManager_LeaderCount%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1682527 10.64.0.132 proxy-server/200 50402 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(jvm_gc_collection_seconds_count%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cgc%3D%22G1%20Young%20Generation%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1730228 10.64.0.132 proxy-server/200 28088 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=increase(jvm_gc_collection_seconds_sum%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cgc%3D%22G1%20Old%20Generation%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:32 1794762 10.64.0.130 proxy-server/200 134717 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_BytesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%20%5B5m%5D))%20by%20(topic)))&start=1556137980&end=1556224440&step=60 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:58 1795967 10.64.0.130 proxy-server/200 34295 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum%20by%20(instance)%20(node_nf_conntrack_entries%7Binstance%3D~%22.*%22%2Ccluster%3D%22eventbus%22%7D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 1867400 10.64.16.24 proxy-server/200 83891 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=jvm_buffer_pool_used_bytes%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:32 2123368 10.64.48.101 proxy-server/200 30626 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(sum%20by%20(instance)%20(kafka_log_Size%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%7D))&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:34:00 2953934 10.64.16.24 proxy-server/200 28262 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=kafka_controller_KafkaController_PreferredReplicaImbalanceCount_Value%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 3336240 10.64.32.67 proxy-server/200 30504 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(kafka_log_LogFlushStats_LogFlushRateAndTimeMs_Count%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 3999222 10.64.16.24 proxy-server/200 83542 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=jvm_memory_bytes_used%7Bcluster%3D%22eventbus%22%2Carea%3D%22heap%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556229600&step=7200 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 4197771 10.64.48.103 proxy-server/200 45598 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(node_disk_reads_completed_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)%20or%20irate(node_disk_reads_completed%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:32 4647238 10.64.48.103 proxy-server/200 129840 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bkafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%5B5m%5D))%20by%20(topic)))&start=1556137980&end=1556224440&step=60 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:34:00 4675382 10.64.48.101 proxy-server/200 83891 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=jvm_buffer_pool_used_bytes%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 7528859 10.64.0.130 proxy-server/200 45608 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(node_disk_reads_completed_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)%20or%20irate(node_disk_reads_completed%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 7586566 10.64.16.22 proxy-server/200 56994 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(node_cpu%7Bcluster%3D%22eventbus%22%2Cmode%3D%22iowait%22%2Cinstance%3D~%22.*.*%22%7D%5B5m%5D))%20by%20(instance)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:38:27 15475441 10.64.16.22 proxy-server/200 57961 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=avg%20by%20(instance)%20(irate(node_cpu%7Bcluster%3D%22eventbus%22%2Cmode!%3D%22idle%22%2Cinstance%3D~%22.*.*%22%7D%5B5m%5D))&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&panelId=24&fullscreen&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 15659306 10.64.48.101 proxy-server/200 2985 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_BytesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548374400&end=1556236800&step=86400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:32 15680869 10.64.0.132 proxy-server/200 23291 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_netstat_Tcp_CurrEstab%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:34:00 15694858 10.64.32.69 proxy-server/200 3025 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548374400&end=1556236800&step=86400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 15757196 10.64.48.103 proxy-server/200 28088 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=increase(jvm_gc_collection_seconds_sum%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cgc%3D%22G1%20Old%20Generation%22%2Cinstance%3D~%22.*.*%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:59 16017768 10.64.32.67 proxy-server/200 49297 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2Ctopic%3D%22%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 16496224 10.64.0.132 proxy-server/200 51422 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(kafka_server_BrokerTopicMetrics_BytesOut_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 16711877 10.64.48.101 proxy-server/200 17612 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_BytesOut_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 17210881 10.64.16.22 proxy-server/200 3058 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_BytesOut_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548374400&end=1556236800&step=86400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:59 17403497 10.64.48.103 proxy-server/200 3015 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548374400&end=1556236800&step=86400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 17934629 10.64.0.132 proxy-server/200 17634 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2Ctopic%3D%22%22%7D%20%5B5m%5D))&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 18270135 10.64.48.103 proxy-server/200 2994 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_BytesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548374400&end=1556236800&step=86400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:32 18431842 10.64.48.103 proxy-server/200 29801 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_Cached_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_Cached%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:58 18503664 10.64.0.132 proxy-server/200 17674 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sum(irate(kafka_server_BrokerTopicMetrics_BytesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D%22%22%7D%20%5B5m%5D))&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 24890074 10.64.0.132 proxy-server/200 57968 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=avg%20by%20(instance)%20(irate(node_cpu%7Bcluster%3D%22eventbus%22%2Cmode!%3D%22idle%22%2Cinstance%3D~%22.*.*%22%7D%5B5m%5D))&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:28 25036402 10.64.32.69 proxy-server/200 59594 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_Cached_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_Cached%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:28 25169711 10.64.0.132 proxy-server/200 72878 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(rate(node_disk_io_time_seconds_total%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)*1000)%20or%20rate(node_disk_io_time_ms%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 25412415 10.64.16.24 proxy-server/200 36065 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(rate(node_disk_io_time_seconds_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)*1000)%20or%20rate(node_disk_io_time_ms%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 25751256 10.64.48.103 proxy-server/200 29807 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 25782217 10.64.32.69 proxy-server/200 32574 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(node_disk_writes_completed_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)%20or%20irate(node_disk_writes_completed%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 51608729 10.64.32.67 proxy-server/200 76382 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(1-%20(node_filesystem_avail_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D))%20or%20(1-%20(node_filesystem_avail%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D))&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 52026727 10.64.32.69 proxy-server/200 34161 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_receive_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%5B5m%5D)%20and%20node_network_receive_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_receive_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%5B5m%5D)%20and%20node_network_receive_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 52933701 10.64.48.101 proxy-server/200 34393 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:28 61108288 10.64.0.130 proxy-server/200 229605 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(1-%20(node_filesystem_avail_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D))%20or%20(1-%20(node_filesystem_avail%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D))&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:28 61492415 10.64.16.24 proxy-server/200 69038 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_transmit_bytes_total%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes_total%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_transmit_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:35:03 79262951 10.64.48.103 proxy-server/200 20286 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=topk(1%2C%20kafka_log_Size%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%7D)%20by%20(instance)%20%3E%200&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 82526168 10.64.32.67 proxy-server/200 40960 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=topk(1%2C%20kafka_log_Size%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%7D)%20by%20(instance)%20%3E%200&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 83850761 10.64.0.132 proxy-server/200 20286 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=topk(1%2C%20kafka_log_Size%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%7D)%20by%20(instance)%20%3E%200&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 85493602 10.64.16.24 proxy-server/200 20286 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bkafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%5B5m%5D))%20by%20(topic)))&start=1548446400&end=1556229600&step=7200 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 86997232 10.64.48.101 proxy-server/200 46166 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=topk(1%2C%20kafka_log_Size%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%7D)%20by%20(instance)%20%3E%200&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 95928829 10.64.16.22 proxy-server/200 45662 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(sum%20by%20(instance)%20(kafka_log_Size%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%7D))&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 96376498 10.64.32.69 proxy-server/200 40960 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bkafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%5B5m%5D))%20by%20(topic)))&start=1548446400&end=1556229600&step=7200 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 102507416 10.64.0.132 proxy-server/200 40960 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_BytesIn_total%7Bcluster%3D%22eventbus%22%2Ckafka_cluster%3D%22main-eqiad%22%2Cinstance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%20%5B5m%5D))%20by%20(topic)))&start=1548446400&end=1556229600&step=7200 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 106339240 10.64.48.101 proxy-server/200 40960 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=sort_desc(topk(3%2C%20sum(irate(kafka_server_BrokerTopicMetrics_MessagesIn_total%7Bkafka_cluster%3D%22main-eqiad%22%2C%20instance%3D~%22.*.*%22%2C%20topic%3D~%22.%2B%22%7D%5B5m%5D))%20by%20(topic)))&start=1548446400&end=1556229600&step=7200 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:28 174948343 10.64.0.132 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(node_memory_MemTotal_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D)%20or%20(node_memory_MemTotal%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:32 178755758 10.64.0.130 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)%20or%20(node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:34:31 184095281 10.64.16.24 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(node_memory_MemTotal_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree_bytes%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D)%20or%20(node_memory_MemTotal%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree%7Bcluster%3D%22kafka_jumbo%22%2Cinstance%3D~%22.*%3A.*%22%7D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:35:31 212999168 10.64.0.132 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)%20or%20(node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)&start=1556137920&end=1556224560&step=240 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1
2019-04-25T20:33:58 267885128 10.64.0.132 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_netstat_Tcp_CurrEstab%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 268417279 10.64.48.101 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_sockstat_TCP_tw%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 273783000 10.64.0.130 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_netstat_Tcp_CurrEstab%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 274279935 10.64.32.69 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_sockstat_TCP_tw%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 275627814 10.64.16.22 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_netstat_Tcp_CurrEstab%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 288419753 10.64.48.101 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_sockstat_TCP_tw%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:03 297309671 10.64.48.103 proxy-server/503 91 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_sockstat_TCP_tw%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - application/json https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 300067101 10.64.16.24 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_Cached_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_Cached%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:00 300071395 10.64.32.69 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(rate(node_disk_io_time_seconds_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)*1000)%20or%20rate(node_disk_io_time_ms%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 300081378 10.64.16.22 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(node_disk_writes_completed_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)%20or%20irate(node_disk_writes_completed%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 300082485 10.64.48.101 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 300086067 10.64.32.67 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)%20or%20(node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 300086099 10.64.32.69 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_receive_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%5B5m%5D)%20and%20node_network_receive_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_receive_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%5B5m%5D)%20and%20node_network_receive_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 300086467 10.64.0.132 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 300086558 10.64.0.130 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(rate(node_disk_io_time_seconds_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)*1000)%20or%20rate(node_disk_io_time_ms%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:01 300089939 10.64.0.132 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(node_disk_writes_completed_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)%20or%20irate(node_disk_writes_completed%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300093493 10.64.48.103 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_Cached_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_Cached%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300093501 10.64.16.24 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers_bytes%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)%20or%20(node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Writeback%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Cached%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_Buffers%7Binstance%3D~%22.*%3A.*%22%7D%20-%20node_memory_MemFree%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300093543 10.64.48.103 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300095245 10.64.48.101 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(1-%20(node_filesystem_avail_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D))%20or%20(1-%20(node_filesystem_avail%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cfstype!~%22(tmpfs%7Crpc_pipefs%7Cdebugfs)%22%7D%20%2F%20node_filesystem_size%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D))&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300095441 10.64.0.130 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_receive_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%5B5m%5D)%20and%20node_network_receive_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_receive_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%5B5m%5D)%20and%20node_network_receive_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300095481 10.64.48.103 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(rate(node_disk_io_time_seconds_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)*1000)%20or%20rate(node_disk_io_time_ms%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300095488 10.64.48.103 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:35:03 300100371 10.64.0.132 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=node_memory_MemTotal_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D%20or%20node_memory_MemTotal%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%7D&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:34:01 300100645 10.64.16.22 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=(irate(node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)%20or%20(irate(node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%5B5m%5D)%20and%20node_network_transmit_bytes%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22(eth%7Ceno).*%22%7D%20%3E%200)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now
2019-04-25T20:33:58 300104187 10.64.32.69 proxy-server/502 463 GET http://grafana.wikimedia.org/api/datasources/proxy/6/api/v1/query_range?query=irate(node_disk_writes_completed_total%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)%20or%20irate(node_disk_writes_completed%7Bcluster%3D%22eventbus%22%2Cinstance%3D~%22.*%3A.*%22%2Cdevice%3D~%22sdb%22%7D%5B5m%5D)&start=1548446400&end=1556236800&step=14400 - text/html https://grafana.wikimedia.org/d/000000027/kafka?refresh=5m&orgId=1&var-datasource=eqiad%20prometheus%2Fops&var-kafka_cluster=main-eqiad&var-cluster=eventbus&var-kafka_broker=All&var-disk_device=sdb&from=now-90d&to=now

Event Timeline

ProdPasteBot changed the title of this paste from untitled to Masterwork From Distant Lands.
CDanis changed the title of this paste from Masterwork From Distant Lands to cdanis@grafana1001.eqiad.wmnet /var/log/apache2 % zgrep '^2019-04-25T20:3' other_vhosts_access.log.4 | sort -g -k +2 | cut -d"$(echo -ne '\t')" -f1-10 | tail -n100 | phaste.Apr 29 2019, 7:40 PM