T266483 run
ActivePublic
Actions

Authored by Kormat on Nov 3 2020, 2:56 PM.

Event Timeline

$ cat ~/bin/nodelist 
#!/bin/bash

nodeset -e -S'\n' $(sudo -H cumin "$@" 2>&1 | head -n2 | tail -n1 | sed 's/\x1b\[[0-9;]*m//g')

$ nodelist '(A:db-section-s5 and A:codfw) and not A:db-store'
db2075.codfw.wmnet
db2089.codfw.wmnet
db2094.codfw.wmnet
db2111.codfw.wmnet
db2113.codfw.wmnet
db2123.codfw.wmnet
db2128.codfw.wmnet
db2137.codfw.wmnet

$ cat bin/host-to-instance
#!/bin/bash

ZARCILLO_HOST=db1115

set -euo pipefail

host="$1"; shift
section="$1"; shift

# Strip off any domain.
host=$(echo $host | cut -f1 -d.)

sudo -H mysql.py -h $ZARCILLO_HOST -A -BN zarcillo -e "select instance from section_instances where section='$section' and instance like '$host%'"

$ host-to-instance db2137 s5
db2137:3315

Kormat edited the content of this paste. (Show Details)Nov 3 2020, 3:26 PM

Kormat edited the content of this paste. (Show Details)Nov 3 2020, 3:36 PM

s5:

kormat@cumin1001:~(0:0)$ mysql.py -h db2123 -e 'show slave hosts'
+-----------+--------------------+------+-----------+
| Server_id | Host               | Port | Master_id |
+-----------+--------------------+------+-----------+
| 180355077 | db2075.codfw.wmnet | 3306 | 180359180 |
| 180359302 | db2137.codfw.wmnet | 3315 | 180359180 |
| 180367384 |                    | 3315 | 180359180 |
| 180367474 | db2128.codfw.wmnet | 3306 | 180359180 |
| 180355081 | db2089.codfw.wmnet | 3315 | 180359180 |
| 180359292 | db2111.codfw.wmnet | 3306 | 180359180 |
| 180363386 |                    | 3315 | 180359180 |
| 180363388 | db2113.codfw.wmnet | 3306 | 180359180 |
+-----------+--------------------+------+-----------+

Kormat mentioned this in T266483: Enable report_host for mariadb.Nov 3 2020, 3:47 PM

s3:

$ mysql.py -h db2105 -e 'show slave hosts'
+-----------+--------------------+------+-----------+
| Server_id | Host               | Port | Master_id |
+-----------+--------------------+------+-----------+
| 180359269 |                    | 3313 | 180355192 |
| 180363447 | db2127.codfw.wmnet | 3306 | 180355192 |
| 180359273 | db2109.codfw.wmnet | 3306 | 180355192 |
| 180367444 |                    | 3306 | 180355192 |
+-----------+--------------------+------+-----------+

s2:

$ mysql.py -h db2107 -e 'show slave hosts'
+-----------+--------------------+------+-----------+
| Server_id | Host               | Port | Master_id |
+-----------+--------------------+------+-----------+
| 180359269 |                    | 3312 | 180359271 |
| 180363445 | db2125.codfw.wmnet | 3306 | 180359271 |
| 180367447 | db2088.codfw.wmnet | 3312 | 180359271 |
| 180363370 | db2138.codfw.wmnet | 3312 | 180359271 |
| 180355191 | db2104.codfw.wmnet | 3306 | 180359271 |
| 180363446 |                    | 3306 | 180359271 |
| 180359272 | db2108.codfw.wmnet | 3306 | 180359271 |
+-----------+--------------------+------+-----------+

s1:

$ mysql.py -h db2112 -e 'show slave hosts'
+-----------+--------------------+------+-----------+
| Server_id | Host               | Port | Master_id |
+-----------+--------------------+------+-----------+
| 180363387 |                    | 3306 | 180363268 |
| 180367493 | db2130.codfw.wmnet | 3306 | 180363268 |
| 180367447 | db2088.codfw.wmnet | 3311 | 180363268 |
| 180355188 |                    | 3311 | 180363268 |
| 180359209 | db2092.codfw.wmnet | 3306 | 180363268 |
| 180355076 | db2071.codfw.wmnet | 3306 | 180363268 |
| 180363307 |                    | 3311 | 180363268 |
| 180355190 | db2103.codfw.wmnet | 3306 | 180363268 |
| 180359205 |                    | 3306 | 180363268 |
| 180355080 | db2085.codfw.wmnet | 3311 | 180363268 |
| 180363399 | db2116.codfw.wmnet | 3306 | 180363268 |
+-----------+--------------------+------+-----------+

s7:

$ mysql.py -h db2118 -e 'show slave hosts'
+-----------+--------------------+------+-----------+
| Server_id | Host               | Port | Master_id |
+-----------+--------------------+------+-----------+
| 180367397 | db2120.codfw.wmnet | 3306 | 180367395 |
| 180355218 | db2122.codfw.wmnet | 3306 | 180367395 |
| 180363435 | db2087.codfw.wmnet | 3317 | 180367395 |
| 180367392 |                    | 3317 | 180367395 |
| 180359208 | db2086.codfw.wmnet | 3317 | 180367395 |
| 180363432 |                    | 3306 | 180367395 |
| 180355206 | db2121.codfw.wmnet | 3306 | 180367395 |
+-----------+--------------------+------+-----------+

s6:

$ mysql.py -h db2129 -e 'show slave hosts'
+-----------+--------------------+------+-----------+
| Server_id | Host               | Port | Master_id |
+-----------+--------------------+------+-----------+
| 180363389 | db2114.codfw.wmnet | 3306 | 180367475 |
| 180363435 | db2087.codfw.wmnet | 3316 | 180367475 |
| 180363307 |                    | 3316 | 180367475 |
| 180367394 | db2117.codfw.wmnet | 3306 | 180367475 |
| 180355081 | db2089.codfw.wmnet | 3316 | 180367475 |
| 180355188 |                    | 3316 | 180367475 |
| 180359187 | db2124.codfw.wmnet | 3306 | 180367475 |
| 180359206 |                    | 3306 | 180367475 |
+-----------+--------------------+------+-----------+

Kormat edited the content of this paste. (Show Details)Nov 4 2020, 9:55 AM

check:

#!/bin/bash

set -euo pipefail

service_name() {
    if echo $inst | grep -q :; then
        echo mariadb@$section;
    else
        echo mariadb
    fi
}

dc=$1; shift

prefix=""
if [ $# -gt 1 ]; then
    prefix="  "
fi

for section in "$@"; do
    if [ -n "$prefix" ]; then
        echo "[?] $section"
    fi
    for fqdn in $(nodelist "(A:db-section-$section and A:$dc)"); do
        inst=$(host-to-instance $fqdn $section)
        rep_host=$(sudo -H mysql.py -h $inst -BN -e 'select @@report_host')
        x=""
        if [ "$rep_host" == "$fqdn" ]; then
            x="x"
        fi
        printf "$prefix[$x] $inst"
        if [ -n "$(sudo -H mysql.py -h $inst -BN -e 'show slave hosts')" ]; then
            printf " has replicas"
        fi
        echo
    done
done

Kormat edited the content of this paste. (Show Details)Nov 4 2020, 10:46 AM

$ ./check codfw s{1..8} x1 m{1,2,3,5} pc{1..3} es{1..5}
[?] s1
  [x] db2071
  [x] db2072 has replicas
  [x] db2085:3311
  [x] db2088:3311
  [x] db2092
  [x] db2094:3311
  [x] db2097:3311
  [x] db2103
  [x] db2112 has replicas
  [x] db2116
  [x] db2130
  [x] db2141:3311
[?] s2
  [x] db2088:3312
  [x] db2095:3312
  [x] db2098:3312
  [x] db2104
  [x] db2107 has replicas
  [x] db2108
  [x] db2125
  [x] db2126 has replicas
  [x] db2138:3312
[?] s3
  [x] db2074 has replicas
  [x] db2094:3313
  [x] db2098:3313
  [x] db2105 has replicas
  [x] db2109
  [x] db2127
[?] s4
  [x] db2073 has replicas
  [x] db2090 has replicas
  [x] db2095:3314
  [x] db2099:3314
  [x] db2106
  [x] db2110
  [x] db2119
  [x] db2136
  [x] db2137:3314
  [x] db2138:3314
  [x] db2139:3314
  [x] db2140
[?] s5
  [x] db2075
  [x] db2089:3315
  [x] db2094:3315
  [x] db2099:3315
  [x] db2111
  [x] db2113
  [x] db2123 has replicas
  [x] db2128 has replicas
  [x] db2137:3315
  [x] db2139:3315
[?] s6
  [x] db2076 has replicas
  [x] db2087:3316
  [x] db2089:3316
  [x] db2095:3316
  [x] db2097:3316
  [x] db2114
  [x] db2117
  [x] db2124
  [x] db2129 has replicas
  [x] db2141:3316
[?] s7
  [x] db2077 has replicas
  [x] db2086:3317
  [x] db2087:3317
  [x] db2095:3317
  [x] db2100:3317
  [x] db2118 has replicas
  [x] db2120
  [x] db2121
  [x] db2122
[?] s8
  [x] db2079 has replicas
  [x] db2080
  [x] db2081
  [x] db2082 has replicas
  [x] db2083
  [x] db2084
  [x] db2085:3318
  [x] db2086:3318
  [x] db2091
  [x] db2094:3318
  [x] db2100:3318
[?] x1
  [x] db2096 has replicas
  [x] db2101:3320
  [x] db2115
  [x] db2131
[?] m1
  [x] db2078:3321
  [x] db2132 has replicas
[?] m2
  [x] db2078:3322
  [x] db2133 has replicas
[?] m3
  [x] db2078:3323
  [x] db2134 has replicas
[?] m5
  [x] db2078:3325
  [x] db2135 has replicas
[?] pc1
  [x] pc2007 has replicas
  [x] pc2010
[?] pc2
  [x] pc2008
[?] pc3
  [x] pc2009
[?] es1
  [x] es2028
  [x] es2030
  [x] es2032
[?] es2
  [x] es2026
  [x] es2031
  [x] es2033
[?] es3
  [x] es2027
  [x] es2029
  [x] es2034
[?] es4
  [x] es2020
  [x] es2021 has replicas
  [x] es2022
[?] es5
  [x] es2023 has replicas
  [x] es2024
  [x] es2025

Kormat edited the content of this paste. (Show Details)Nov 5 2020, 9:23 AM

Kormat edited the content of this paste. (Show Details)Nov 5 2020, 9:27 AM

Kormat edited the content of this paste. (Show Details)Nov 5 2020, 9:49 AM

	#!/bin/bash

	set -euo pipefail

	IGNORE_REPLICAS=${IGNORE_REPLICAS:-}

	service_suffix() {
	if echo $inst \| grep -q :; then
	echo @$section;
	fi
	}

	section=$1; shift
	dc=$1; shift

	for fqdn in $(nodelist "(A:db-section-$section and A:$dc) and not A:db-store"); do
	inst=$(host-to-instance $fqdn $section)
	echo "====> $inst"
	rep_host=$(sudo -H mysql.py -h $inst -BN -e 'select @@report_host')
	if [ "$rep_host" == "$fqdn" ]; then
	echo "Already done"
	continue
	fi
	if [ -z "$IGNORE_REPLICAS" ]; then
	if [ -n "$(sudo -H mysql.py -h $inst -BN -e 'show slave hosts')" ]; then
	echo "Not touching, has replicas"
	continue
	fi
	fi

	SSH_AUTH_SOCK=/run/keyholder/proxy.sock ssh root@$fqdn systemctl restart mariadb$(service_suffix)
	SSH_AUTH_SOCK=/run/keyholder/proxy.sock ssh root@$fqdn systemctl restart prometheus-mysqld-exporter$(service_suffix)

	if [ -n "$(sudo -H mysql.py -h $inst -BN -e 'show slave status')" ]; then
	sudo -H mysql.py -h $inst -e 'start slave'
	fi
	done

T266483 runActivePublicActions

Event Timeline

T266483 run
ActivePublic
Actions