Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Paste
P6846
(An Untitled Masterwork)
Active
Public
Actions
Authored by
Gehel
on Mar 14 2018, 2:06 PM.
Edit Paste
Archive Paste
View Raw File
Subscribe
Mute Notifications
Award Token
Flag For Later
Tags
None
Referenced Files
F15406113:
Mar 14 2018, 2:06 PM
2018-03-14 14:06:45 (UTC+0)
Subscribers
None
#!/bin/bash
#
# WMF Update production known hosts
#
# - Populate a known_hosts file with all the production hosts and services
# in the Wikimedia Foundation production infrastructure:
# - sync all the known hosts from a bastion
# - optionally generate known hosts for services defined as CNAMEs in the
# DNS repository
# - Keeps a backup file with the previous known hosts
# - Show a diff between the new known hosts and the current ones
#
# It saves the known hosts into KNOWN_HOST_FILE, adjust this and/or the
# UserKnownHostsFile parameter in your ~/.ssh/config in order for them to
# match.
#
# By default only the hosts from the choosen BASTION_HOST known_hosts file
# will be imported, cleaning the hostname (not the FQDN) to ease the auto-
# completion when ssh-ing.
#
# It accept one positional argument that, if specified, must be the path to
# a local clone of the Operations DNS repository:
# https://gerrit.wikimedia.org/r/operations/dns
# In this case also the services defined as CNAMEs in the wikimedia.org and
# wmnet zone files will be added with the identity of the target host, if
# that is found in the known_hosts file, skipping the missing ones.
#
# Example usage:
# wmf-update-prod-known-hosts
# wmf-update-prod-known-hosts ~/code/gerrit.w.o/dns
#
# Author: Riccardo Coccioli <rcoccioli@wikimedia.org>
# Date: 2017-06-21
# Dependencies: colordiff
#####################
set
-e
DNS_REPO_PATH
=
"
${
1
}
"
KNOWN_HOSTS_PATH
=
"
${
HOME
}
/.ssh/known_hosts.d"
KNOWN_HOST_FILE
=
"
${
KNOWN_HOSTS_PATH
}
/wmf-prod"
BASTION_HOST
=
"bast1001.wikimedia.org"
function
parse_line
()
{
local
line
=
"
${
1
}
"
local
domain
=
"
${
2
}
"
local
name
local
target
local
found
name
=
"
$(
echo
"
${
line
}
"
|
cut -d
' '
-f1
)
"
target
=
"
$(
echo
"
${
line
}
"
|
cut -d
' '
-f2
)
"
sep
=
"\."
if
[[
"
${
target
: -1
}
"
==
'.'
]]
;
then
target
=
"
${
target
%?
}
"
sep
=
","
fi
set
+e
found
=
$(
grep -c
"^
${
target
}${
sep
}
"
"
${
KNOWN_HOST_FILE
}
.new"
)
set
-e
if
[[
"
${
found
}
"
-eq
"0"
||
"
${
found
}
"
-gt
"1"
]]
;
then
>
&
2
echo
"Skipping '
${
target
}
' CNAME target, found
${
found
}
/1 matches"
return
fi
grep
"^
${
target
}${
sep
}
"
"
${
KNOWN_HOST_FILE
}
.new"
|
awk -v
name
=
"
${
name
}
"
-v
domain
=
"
${
domain
}
"
'{ printf name"."domain; for (i = 2; i <= NF; i++) printf FS$i; print NL }'
}
function
extract_cnames_from_zone
()
{
local
zone_file
local
origin
local
boundaries
local
start
local
end
local
domain
zone_file
=
"
${
1
}
"
if
[[
! -f
"
${
zone_file
}
"
]]
;
then
>
&
2
echo
"Unable to find zone file
${
zone_file
}
, skipping..."
return
fi
origin
=
"
${
2
}
"
if
[[
-n
"
${
origin
}
"
]]
;
then
boundaries
=
"
$(
grep -n
"\$ORIGIN"
"
${
zone_file
}
"
|
grep -A
1
"\$ORIGIN
${
origin
}
\.
$
"
)
"
start
=
$(
echo
"
${
boundaries
}
"
|
head -n1
|
cut -d
':'
-f1
)
end
=
$(
echo
"
${
boundaries
}
"
|
tail -n1
|
cut -d
':'
-f1
)
domain
=
"
${
origin
}
"
head -n
"
${
end
}
"
"
${
zone_file
}
"
|
tail -n
"
$((
end
-
start
))
"
|
grep
" CNAME "
|
awk
'{ print $1, $5 }'
|
while
read
-r line
;
do
parse_line
"
${
line
}
"
"
${
domain
}
"
>>
"
${
KNOWN_HOST_FILE
}
.new"
done
else
domain
=
"
$(
basename
"
${
zone_file
}
"
)
"
grep
" CNAME "
"
${
zone_file
}
"
|
awk
'{ print $1, $5 }'
|
while
read
-r line
;
do
parse_line
"
${
line
}
"
"
${
domain
}
"
>>
"
${
KNOWN_HOST_FILE
}
.new"
done
fi
}
# Get new known hosts
ssh
"
${
BASTION_HOST
}
"
'cat /etc/ssh/ssh_known_hosts'
>
"
${
KNOWN_HOST_FILE
}
.new"
# Remove the non-FQDN hostnames to avoid multiple autocompletions
awk -F
','
'{ printf $1; for (i = 3; i <= NF; i++) printf FS$i; print NL }'
"
${
KNOWN_HOST_FILE
}
.new"
>
"
${
KNOWN_HOST_FILE
}
.new.clean"
mv -f
"
${
KNOWN_HOST_FILE
}
.new.clean"
"
${
KNOWN_HOST_FILE
}
.new"
if
[[
-n
"
${
DNS_REPO_PATH
}
"
]]
;
then
extract_cnames_from_zone
"
${
DNS_REPO_PATH
}
/templates/wikimedia.org"
extract_cnames_from_zone
"
${
DNS_REPO_PATH
}
/templates/wmnet"
"eqiad.wmnet"
fi
echo
"==== DIFFERENCES ===="
colordiff --fakeexitcode
"
${
KNOWN_HOST_FILE
}
"
"
${
KNOWN_HOST_FILE
}
.new"
echo
"====================="
echo
"Going from
$(
wc -l
"
${
KNOWN_HOST_FILE
}
"
)
to
$(
wc -l
"
${
KNOWN_HOST_FILE
}
.new"
)
known hosts and services"
mv -f
"
${
KNOWN_HOST_FILE
}
"
"
${
KNOWN_HOST_FILE
}
.old"
&&
mv
"
${
KNOWN_HOST_FILE
}
.new"
"
${
KNOWN_HOST_FILE
}
"
echo
"Backup file is
${
KNOWN_HOST_FILE
}
.old"
exit
0
Event Timeline
Gehel
created this paste.
Mar 14 2018, 2:06 PM
2018-03-14 14:06:45 (UTC+0)
Log In to Comment