# schedule and announce downtime
# on gerrit1001: shortly before the scheduled downtime:
# on gerrit1001, as root, in a screen: rsync -avp --delete --bwlimit=5m /var/lib/gerrit2/review_site/ rsync://gerrit1003.wikimedia.org/gerrit-var-lib/ (raise bwlimit !?)
# on gerrit1001, as root, in a screen: rsync -avp --delete --bwlimit=5m /srv/gerrit/ rsync://gerrit1003.wikimedia.org/gerrit-data/ (raise bwlimit !?)
# scheduled downtime begins / IRC announcement
# on cumin1001:sudo cookbook sre.hosts.downtime -r 'maintenance' -D 30 gerrit1001.wikimedia.org
# on cumin1001:sudo cookbook sre.hosts.downtime -r 'maintenance' -H 1 gerrit1003.wikimedia.org
# on gerrit1003: disable puppet; stop gerrit? (sudo disable-puppet 'gerrit maintenance'; systemctl stop gerrit)
# merge DNS change that removes gerrit-new and switches IP of gerrit.wikimedia.org - in web UI of gerrit(-old)
# run authdns-update on ns0.wikimedia.org, see the diff but do NOT commit yet
# on gerrit1001: disable puppet; stop gerrit! (sudo disable-puppet 'gerrit maintenance'; systemctl stop gerrit)
# on gerrit1001, as root, in a screen: rsync -avp --delete --bwlimit=10m /var/lib/gerrit2/review_site/ rsync://gerrit1003.wikimedia.org/gerrit-var-lib/
# on gerrit1001, as root, in a screen: rsync -avp --delete --bwlimit=10m /srv/gerrit/ rsync://gerrit1003.wikimedia.org/gerrit-data/
# on gerrit1003: cd /var/lib/gerrit2/; chown -R gerrit2:gerrit2 gerrit/
# on gerrit1003: cd /srv/; chown -R gerrit2:gerrit2 gerrit/
# on gerrit1003: start gerrit
# say "yes" to authdns-update and actually merge DNS change that removes gerrit-new and switches IP of gerrit.wikimedia.org
# wait 5 minutes
# ..test https
# ..test ssh
# announce downtime is over
# ensure old host has puppet disabled and/or services masked
# grace period
# decom old host