Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F13738525
Update production known hosts
No One
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Authored By
Volans
Feb 14 2018, 12:09 PM
2018-02-14 12:09:29 (UTC+0)
Size
5 KB
Referenced Files
None
Subscribers
None
Update production known hosts
View Options
#!/bin/bash
##############################################################################
# WMF Update production known hosts
#
# DESCRIPTION:
# - Populate a known_hosts file with all the production hosts and services
# in the Wikimedia Foundation production infrastructure for easy
# autocompletion while keeping StrictHostKeyChecking active:
# - sync all the known hosts from a bastion
# - clean the hostname without FQDN in it
# - optionally generate known hosts for services defined as CNAMEs in the
# DNS repository, see PARAMS below. This allows for the autocompletion of
# active/passive services like icinga.wikimedia.org.
# - Keeps a backup file with the previous known hosts
# - Show a diff between the new known hosts and the current ones
#
# It saves the known hosts into KNOWN_HOST_FILE, adjust this and/or the
# UserKnownHostsFile parameter in your ~/.ssh/config in order for them to
# match. A warning will be shown if they don't match.
#
# By default only the hosts from the choosen BASTION_HOST known_hosts file
# will be imported, cleaning the hostname (not the FQDN) to ease the auto-
# completion when ssh-ing.
#
# PARAMS:
# It accept one positional argument that, if specified, must be the path to
# a local clone of the Operations DNS repository, (either from Gerrit or from
# GitHub):
# https://gerrit.wikimedia.org/r/operations/dns
# In this case also the services defined as CNAMEs in the wikimedia.org and
# wmnet zone files will be added with the identity of the target host, if
# that is found in the known_hosts file, skipping the missing ones.
#
# USAGE:
# wmf-update-prod-known-hosts [PATH_TO_DNS_REPOSITORY]
#
# Author: Riccardo Coccioli <rcoccioli@wikimedia.org>
# Date: 2017-06-21
# Dependencies: colordiff
# Version: 1.0
# License: GPLv3+
##############################################################################
set
-e
DNS_REPO_PATH
=
"
${
1
}
"
KNOWN_HOSTS_PATH
=
"
${
HOME
}
/.ssh/known_hosts.d"
KNOWN_HOST_FILE
=
"
${
KNOWN_HOSTS_PATH
}
/wmf-prod"
BASTION_HOST
=
"bast1001.wikimedia.org"
if
[[
! -d
"
${
KNOWN_HOSTS_PATH
}
"
]]
;
then
echo
"ERROR: KNOWN_HOSTS_PATH '
${
KNOWN_HOSTS_PATH
}
' is not a directory, you might want to adjust the constant in the script or create it"
exit
1
fi
if
[[
-n
"
${
DNS_REPO_PATH
}
"
]]
;
then
if
[[
! -d
"
${
DNS_REPO_PATH
}
"
]]
;
then
echo
"ERROR: DNS_REPO_PATH '
${
DNS_REPO_PATH
}
' is not a directory"
exit
2
fi
if
! git -C
"
${
DNS_REPO_PATH
}
"
remote -v
|
egrep
'(gerrit.wikimedia.org|github.com\/wikimedia)'
|
grep -cq
'operations[/-]dns'
;
then
echo
"ERROR: DNS_REPO_PATH '
${
DNS_REPO_PATH
}
' doesn't seems to be a checkout of the operations/dns repository"
exit
3
fi
fi
function
parse_line
()
{
local
line
=
"
${
1
}
"
local
domain
=
"
${
2
}
"
local
name
local
target
local
found
name
=
"
$(
echo
"
${
line
}
"
|
cut -d
' '
-f1
)
"
target
=
"
$(
echo
"
${
line
}
"
|
cut -d
' '
-f2
)
"
sep
=
"\."
if
[[
"
${
target
: -1
}
"
==
'.'
]]
;
then
target
=
"
${
target
%?
}
"
sep
=
","
fi
set
+e
found
=
$(
grep -c
"^
${
target
}${
sep
}
"
"
${
KNOWN_HOST_FILE
}
.new"
)
set
-e
if
[[
"
${
found
}
"
-eq
"0"
||
"
${
found
}
"
-gt
"1"
]]
;
then
>
&
2
echo
"Skipping '
${
target
}
' CNAME target, found
${
found
}
/1 matches"
return
fi
grep
"^
${
target
}${
sep
}
"
"
${
KNOWN_HOST_FILE
}
.new"
|
awk -v
name
=
"
${
name
}
"
-v
domain
=
"
${
domain
}
"
'{ printf name"."domain; for (i = 2; i <= NF; i++) printf FS$i; print NL }'
}
function
extract_cnames_from_zone
()
{
local
zone_file
local
origin
local
boundaries
local
start
local
end
local
domain
zone_file
=
"
${
1
}
"
if
[[
! -f
"
${
zone_file
}
"
]]
;
then
>
&
2
echo
"Unable to find zone file
${
zone_file
}
, skipping..."
return
fi
origin
=
"
${
2
}
"
if
[[
-n
"
${
origin
}
"
]]
;
then
boundaries
=
"
$(
grep -n
"\$ORIGIN"
"
${
zone_file
}
"
|
grep -A
1
"\$ORIGIN
${
origin
}
\.
$
"
)
"
start
=
$(
echo
"
${
boundaries
}
"
|
head -n1
|
cut -d
':'
-f1
)
end
=
$(
echo
"
${
boundaries
}
"
|
tail -n1
|
cut -d
':'
-f1
)
domain
=
"
${
origin
}
"
head -n
"
${
end
}
"
"
${
zone_file
}
"
|
tail -n
"
$((
end
-
start
))
"
|
awk
'/ CNAME / { print $1, $5 }'
|
while
read
-r line
;
do
parse_line
"
${
line
}
"
"
${
domain
}
"
>>
"
${
KNOWN_HOST_FILE
}
.new"
done
else
domain
=
"
$(
basename
"
${
zone_file
}
"
)
"
awk
'/ CNAME / { print $1, $5 }'
"
${
zone_file
}
"
|
while
read
-r line
;
do
parse_line
"
${
line
}
"
"
${
domain
}
"
>>
"
${
KNOWN_HOST_FILE
}
.new"
done
fi
}
# Get new known hosts
echo
"===> SSHing to
${
BASTION_HOST
}
(if a smart card input is needed, check it now)"
ssh
"
${
BASTION_HOST
}
"
'cat /etc/ssh/ssh_known_hosts'
>
"
${
KNOWN_HOST_FILE
}
.new"
# Remove the non-FQDN hostnames to avoid multiple autocompletions
awk -F
','
'{ printf $1; for (i = 3; i <= NF; i++) printf FS$i; print NL }'
"
${
KNOWN_HOST_FILE
}
.new"
>
"
${
KNOWN_HOST_FILE
}
.new.clean"
mv -f
"
${
KNOWN_HOST_FILE
}
.new.clean"
"
${
KNOWN_HOST_FILE
}
.new"
if
[[
-n
"
${
DNS_REPO_PATH
}
"
]]
;
then
extract_cnames_from_zone
"
${
DNS_REPO_PATH
}
/templates/wikimedia.org"
extract_cnames_from_zone
"
${
DNS_REPO_PATH
}
/templates/wmnet"
"eqiad.wmnet"
fi
PREV_COUNT
=
0
PREV_FILE
=
/dev/null
if
[[
-f
"
${
KNOWN_HOST_FILE
}
"
]]
;
then
PREV_COUNT
=
"
$(
wc -l
"
${
KNOWN_HOST_FILE
}
"
)
"
PREV_FILE
=
"
${
KNOWN_HOST_FILE
}
"
fi
echo
"==== DIFFERENCES ===="
colordiff --fakeexitcode
"
${
PREV_FILE
}
"
"
${
KNOWN_HOST_FILE
}
.new"
echo
"====================="
echo
"Going from
${
PREV_COUNT
}
to
$(
wc -l
"
${
KNOWN_HOST_FILE
}
.new"
)
known hosts and services"
if
[[
-f
"
${
KNOWN_HOST_FILE
}
"
]]
;
then
mv -vf
"
${
KNOWN_HOST_FILE
}
"
"
${
KNOWN_HOST_FILE
}
.old"
echo
"Backup file is
${
KNOWN_HOST_FILE
}
.old"
fi
mv -v
"
${
KNOWN_HOST_FILE
}
.new"
"
${
KNOWN_HOST_FILE
}
"
echo
"New file generated at
${
KNOWN_HOST_FILE
}
"
if
! egrep -cq
"UserKnownHostsFile .*/wmf-prod( |
$
)"
"
${
HOME
}
/.ssh/config"
;
then
echo
"WARNING: You may need to add/update 'UserKnownHostsFile
${
KNOWN_HOST_FILE
}
' to your ~/.ssh/config"
fi
exit
0
File Metadata
Details
Attached
Mime Type
text/plain; charset=utf-8
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
5525009
Default Alt Text
Update production known hosts (5 KB)
Attached To
Mode
P5608 Update production known hosts
Attached
Detach File
Event Timeline
Log In to Comment