Page MenuHomePhabricator
Paste P10810

fix-workers.sh
ActivePublic

Authored by Bstorm on Mar 28 2020, 12:40 AM.
Tags
None
Referenced Files
F31706202: raw.txt
Mar 28 2020, 12:40 AM
Subscribers
None
#!/usr/local/bin/bash
set -Eeuo pipefail
workers=(toolsbeta-test-k8s-worker-1.toolsbeta.eqiad.wmflabs toolsbeta-test-k8s-worker-2.toolsbeta.eqiad.wmflabs toolsbeta-test-k8s-worker-3.toolsbeta.eqiad.wmflabs)
controlnode=toolsbeta-test-k8s-control-1.toolsbeta.eqiad.wmflabs
SSHCMD="/usr/bin/ssh -l root"
for worker in ${workers[*]}
do
node=${worker%%.*}
echo "Draining $node"
$SSHCMD $controlnode "kubectl drain --force --ignore-daemonsets --delete-local-data $node"
echo "Stopping the services"
$SSHCMD "$worker" "systemctl stop docker"
$SSHCMD "$worker" "systemctl stop kubelet"
echo "destroying /var/lib/docker"
$SSHCMD "$worker" "rm -rf /var/lib/docker"
echo "bringing back puppet"
$SSHCMD "$worker" "puppet agent --enable"
$SSHCMD "$worker" "puppet agent -t" || true
$SSHCMD $controlnode "kubectl uncordon $node"
echo "***${node} is done***"
done