At approximately 17:27 UTC on Jan 23, 2017 Cassandra on restbase200{4,7}.codfw.wmnet exited with OOM exceptions.
$ for i in 2004 2007; do echo "$i: "; ssh restbase$i.codfw.wmnet -- "sudo find /srv/cassandra-* -maxdepth 1 -name '*.hprof' -exec ls -lh {} \;"; done 2004: -rw------- 1 cassandra cassandra 8.3G Jan 23 17:27 /srv/cassandra-a/java_pid6608.hprof 2007: -rw-r--r-- 1 cassandra cassandra 9.0G Jan 23 17:26 /srv/cassandra-b/java_pid22594.hprof