(An Untitled Masterwork)
ActivePublic
Actions

Authored by thcipriani on Feb 19 2019, 2:53 PM.

Tags

None

Referenced Files

	F28246222:
	Feb 19 2019, 2:53 PM

Subscribers

None

	#!/bin/bash
	#
	# Sets up an environment with multiple LXC containers for testing deployment
	# tooling. See the README for details.
	#

	set -e

	while [ -n "$1" ]; do
	case "$1" in
	'debug')
	set -o xtrace
	;;
	esac
	shift
	done

	PACKAGES=(
	apache2
	apache2-utils
	bridge-utils
	btrfs-tools
	git
	git-annex
	git-buildpackage
	libssl-dev
	libvirt-clients
	libvirt-daemon-system
	lxc
	php-cli
	php-memcached
	php-redis
	php-mbstring
	php-xml
	python3-pip
	python-pip
	rsync
	uidmap
	vim
	)

	TARGET_PACKAGES=(
	curl
	git
	git-annex
	nagios-nrpe-server
	netcat
	python
	python-jinja2
	python-psutil
	python-requests
	python-semver
	python-yaml
	python-conftool
	python-configparser
	python-pygments
	python-pip
	sudo
	tree
	)

	PIP_PACKAGES=()

	GITFAT_URL=https://raw.githubusercontent.com/jedbrown/git-fat/master/git-fat

	SCAP_REPO=https://phabricator.wikimedia.org/diffusion/MSCA/scap.git
	MW_CONFIG=https://gerrit.wikimedia.org/r/operations/mediawiki-config
	MW_CORE=https://gerrit.wikimedia.org/r/mediawiki/core

	CONTAINER_PREFIX=scap-target
	BASE_CONTAINER="$CONTAINER_PREFIX-base"
	PASSWORD=vagrant

	DEPLOY_DIR=/srv/deployment/mockbase/deploy

	VAGRANT_UID="$(id -u vagrant)"
	VAGRANT_GID="$(id -g vagrant)"

	L10NUPDATE_ID=1002

	# Function for doing stuff to our base container
	lxc() {
	local cmd=$1
	shift 1

	"lxc-$cmd" -n "$BASE_CONTAINER" "$@"

	return $?
	}

	lxc_wait_for_ip() {
	echo "Waiting for $1 to get an IP"
	timeout 30s bash <<END
	while [ -z "\$(lxc-info -i -n '$1')" ]; do
	sleep 0.2
	done
	END
	}

	lxc_wait_for_ssh() {
	lxc_wait_for_ip "$1"

	echo "Waiting for SSH on $1"
	timeout 30s bash <<END
	while ! nc -z $1 22; do
	sleep 0.2
	done
	END
	}

	# Prelim setup
	apt-get -y update
	apt-get -y install git curl

	# Get latest wikiversion from prod
	get_wikiversion() {
	version=$(git ls-remote -h "$MW_CORE" \| \
	awk '/wmf\// {gsub("refs/heads/wmf/", ""); print $2}' \| \
	sort -Vr \| \
	head -1)

	printf '%s\n' "$version"
	}

	MW_VERSION=$(get_wikiversion)

	# Set up apt cache and install packages
	if ! [ -f /etc/apt/apt.conf.d/20shared-cache ]; then
	mkdir -p /vagrant/cache/apt
	echo 'Dir::Cache::archives "/vagrant/cache/apt";' > /etc/apt/apt.conf.d/20shared-cache
	fi

	if ! [ -f /etc/profile.d/goawaypyc.sh ]; then
	echo 'export PYTHONDONTWRITEBYTECODE=1' > /etc/profile.d/goawaypyc.sh
	fi

	curl -sLo /tmp/wikikey "https://wikitech.wikimedia.org/w/index.php?title=APT_repository/Stretch-Key&action=raw"
	apt-key add /tmp/wikikey

	if ! [ -f /etc/apt/sources.list.d/wikimedia.list ]; then
	cat > /etc/apt/sources.list.d/wikimedia.list <<END
	## Wikimedia APT repository
	deb http://apt.wikimedia.org/wikimedia stretch-wikimedia main
	deb-src http://apt.wikimedia.org/wikimedia stretch-wikimedia main
	END

	cat > /etc/apt/preferences.d/wikimedia.pref <<END
	Package: *
	Pin: release o=Wikimedia
	Pin-Priority: 1001
	END
	fi

	apt-get -y update
	apt-get -y install "${PACKAGES[@]}" "${TARGET_PACKAGES[@]}"

	# Install pip packages
	for package in "${PIP_PACKAGES[@]}"; do
	if ! pip2 show "$package" 2> /dev/null \| grep -q .; then
	pip2 install "$package"
	fi
	done
	for package in "${PIP_PACKAGES[@]}"; do
	if ! pip3 show "$package" 2> /dev/null \| grep -q .; then
	pip3 install "$package"
	fi
	done

	# Create bridge interface
	if ! virsh net-list \| grep -q default; then
	virsh net-start default
	virsh net-autostart default
	fi

	# Configure host DNS to resolve from dnsmasq first
	if ! cmp -s {/vagrant/files/dhcp,/etc/dhcp/dhclient-enter-hooks.d}/prefer-local-nameserver; then
	echo 'Configuring host DNS to resolve local container names'
	install -o root -m 0755 /vagrant/files/dhcp/prefer-local-nameserver /etc/dhcp/dhclient-enter-hooks.d/
	service networking reload
	fi

	# Clone scap into /scap if it's not already cloned
	if ! [ -d /scap/.git ]; then
	echo "Cloning $SCAP_REPO to /scap"
	git clone -q "$SCAP_REPO" /scap
	fi

	# Add scap project directory to our PATH
	echo 'PATH=/srv/deployment/scap/scap/bin:"$PATH"' > /etc/profile.d/scap.sh

	# Install git-fat
	if ! [ -f /usr/local/bin/git-fat ]; then
	echo 'Installing git-fat'
	curl -s "$GITFAT_URL" > /usr/local/bin/git-fat
	chmod +x /usr/local/bin/git-fat
	fi

	# Create a git-fat store
	if ! [ -d /home/vagrant/fat-store ]; then
	mkdir /home/vagrant/fat-store
	chown vagrant:vagrant /home/vagrant/fat-store
	fi

	# Add helper functions to profile
	install -o root -m 0644 /vagrant/files/helper.sh /etc/profile.d/

	# Create deployment environment
	if ! [ -d /srv/deployment ]; then
	mkdir -p /srv/deployment
	chown vagrant:vagrant /srv/deployment
	fi

	# Set up Apache to host it
	if ! cmp -s {/etc/apache2/sites-available,/vagrant/files/apache}/deployment.conf; then
	echo 'Setting up Apache vhost for deployment git repos'
	install -o root -m 0644 /vagrant/files/apache/deployment.conf /etc/apache2/sites-available/
	a2ensite deployment
	service apache2 reload
	fi

	# Create an SSH key for the vagrant user (authorized on each container later)
	if ! [ -f /home/vagrant/.ssh/id_rsa ]; then
	sudo -u vagrant ssh-keygen -t rsa -f /home/vagrant/.ssh/id_rsa -N ''
	fi

	# Authorize our own SSH host key (so we can set up git-fat objects)
	if ! grep -q '192.168.122.1' /etc/ssh/ssh_known_hosts; then
	echo 'Authorizing our own SSH host key'
	ssh-keyscan -H -t ecdsa 192.168.122.1 >> /etc/ssh/ssh_known_hosts 2> /dev/null
	fi

	# Authorize connecting to ourselves (so we can set up git-fat objects)
	if ! grep -q "vagrant@$HOSTNAME$" /home/vagrant/.ssh/authorized_keys; then
	echo 'Authorizing SSH access to 192.168.122.1'
	cat /home/vagrant/.ssh/id_rsa.pub >> /home/vagrant/.ssh/authorized_keys
	fi

	# Prepare central mockbase repo
	if ! [ -d /var/lib/git/mockbase ]; then
	echo 'Preparing central mockbase repo'
	sudo -su vagrant git config --global user.name vagrant
	sudo -su vagrant git config --global user.email vagrant@localhost

	dir="$(mktemp -d)"
	rsync --exclude=*.swp --exclude=.git --delete -qa /vagrant/files/mockbase/ "$dir/"

	pushd "$dir"
	sudo -su vagrant git init ./
	sudo -su vagrant git add --all :/
	sudo -su vagrant git commit -m 'initial commit'
	popd

	git clone -q --bare "$dir" /var/lib/git/mockbase
	pushd /var/lib/git/mockbase
	git update-server-info
	popd
	rm -rf "$dir"
	fi

	# Prepare deploy repo
	if ! [ -d "$DEPLOY_DIR" ]; then
	echo 'Cloning git repos into the deployment directory'
	sudo -su vagrant mkdir -p "$DEPLOY_DIR"
	sudo -su vagrant rsync --exclude=*.swp --exclude=.git --delete -qr /vagrant/files/deploy/ "$DEPLOY_DIR/"
	fi

	if ! [ -d "$DEPLOY_DIR/.git" ]; then
	pushd "$DEPLOY_DIR"
	# Commit the initial repository state
	sudo -su vagrant git init ./
	sudo -su vagrant git fat init ./
	sudo -su vagrant git add --all :/
	sudo -su vagrant git submodule -q add http://192.168.122.1/git/mockbase
	sudo -su vagrant git commit -m 'initial commit'

	# Create a couple of git-fat objects
	# NOTE the .gitfat and .gitattributes files should already be configured
	head -c 10m /dev/urandom \| sudo -su vagrant tee big1.bin > /dev/null
	head -c 20m /dev/urandom \| sudo -su vagrant tee big2.bin > /dev/null
	sudo -su vagrant git add ./*.bin
	sudo -su vagrant git commit -m 'added binary files'
	sudo -su vagrant git fat push

	# Add a remote so we don't choke on `git rev-parse origin/master` later
	sudo -su vagrant git remote add origin "$DEPLOY_DIR/.git"
	sudo -su vagrant git fetch origin
	popd
	fi

	# Add users for mwdeploy testing
	if ! getent group l10nupdate; then
	addgroup --gid $L10NUPDATE_ID l10nupdate
	fi

	if ! getent passwd l10nupdate; then
	adduser --system \
	--home '/home/l10nupdate' \
	--shell '/bin/bash' \
	--uid $L10NUPDATE_ID \
	--gid $L10NUPDATE_ID \
	l10nupdate
	fi

	if ! getent group mwdeploy; then
	addgroup mwdeploy
	fi

	if ! getent passwd mwdeploy; then
	adduser --system \
	--home '/home/mwdeploy' \
	--shell '/bin/bash' \
	mwdeploy
	fi

	# Use the same ssh key for all users
	for user in l10nupdate mwdeploy; do
	if [ ! -f /home/"$user"/.ssh/id_rsa ]; then
	mkdir -p /home/"$user"/.ssh
	cp /home/vagrant/.ssh/id_rsa* /home/"$user"/.ssh/
	chown -R "$user":"$user" /home/"$user"/.ssh
	chmod 700 /home/"$user"/.ssh
	chmod 600 /home/"$user"/.ssh/id_rsa
	fi
	done

	# Allow networked containers
	cat > /etc/lxc/default.conf <<-end
	lxc.network.type = veth
	lxc.network.flags = up
	lxc.network.link = virbr0
	lxc.network.hwaddr = 00:FF:AA:00:00:xx
	lxc.network.ipv4 = 0.0.0.0/24
	end

	# Move /var/lib/lxc to a btrfs mount so we can do snapshots
	if [ -z "$(awk '/^[^#]/ && $2 == "/var/lib/lxc" { print $2 }' /etc/fstab)" ]; then
	truncate -s 3G /var/local/lxc.img
	dev=$(losetup --show -f /var/local/lxc.img)
	mkfs.btrfs "$dev"
	sync
	losetup -d "$dev"
	echo '/var/local/lxc.img /var/lib/lxc btrfs loop 0 0' >> /etc/fstab
	fi

	# Make sure /var/lib/lxc is mounted
	if ! mountpoint -q /var/lib/lxc; then
	mount /var/lib/lxc
	fi

	# Restore the LXC download cache if there is one
	if [ -d /vagrant/cache/lxc/download ]; then
	rsync -qrlt /vagrant/cache/lxc/download/ /var/cache/lxc/download/
	fi

	# Set up base LXC container that we'll use as a template for clones
	if [ -z "$(lxc-ls -1 $BASE_CONTAINER)" ]; then
	lxc create -t download -B btrfs -- -d debian -r stretch -a amd64

	BASE_ROOT=/var/lib/lxc/$BASE_CONTAINER/rootfs

	echo 'Updating LXC download cache'
	mkdir -p /vagrant/cache/lxc/download
	rsync -qrlt /var/cache/lxc/download/ /vagrant/cache/lxc/download/

	mkdir -p $BASE_ROOT/srv/deployment/scap/scap

	echo 'Setting up mockbase in base container'
	mkdir -p "$BASE_ROOT/${DEPLOY_DIR}-cache"
	chown -R vagrant:vagrant "$BASE_ROOT/$(dirname $DEPLOY_DIR)"
	mkdir -p $BASE_ROOT/etc/mockbase
	cp /vagrant/files/mockbase/mockbase.service $BASE_ROOT/etc/mockbase/
	cat > $BASE_ROOT/etc/mockbase/config-vars.yaml <<END
	---
	foo: bar
	END

	cat > $BASE_ROOT/etc/apt/sources.list.d/wikimedia.list <<END
	## Wikimedia APT repository
	deb http://apt.wikimedia.org/wikimedia stretch-wikimedia main
	deb-src http://apt.wikimedia.org/wikimedia stretch-wikimedia main
	END

	cat > $BASE_ROOT/etc/apt/preferences.d/wikimedia.pref <<END
	Package: *
	Pin: release o=Wikimedia
	Pin-Priority: 1001
	END

	cp /tmp/wikikey $BASE_ROOT/tmp/wikikey

	echo 'Starting base container'
	lxc start -d
	lxc wait -s RUNNING

	# Wait for the network to come up
	lxc_wait_for_ip $BASE_CONTAINER

	echo 'Installing scap dependencies into base container'
	lxc attach -- apt-get -y update
	lxc attach -- apt-get -y --force-yes install apt-utils openssh-server
	lxc attach -- apt-get -y --force-yes install "${TARGET_PACKAGES[@]}"
	lxc attach -- pip install -U "${TARGET_PIP_PACKAGES[@]}"

	echo 'Setting up base container users'
	lxc attach -- groupadd -g "$VAGRANT_GID" vagrant
	lxc attach -- groupadd -g "$L10NUPDATE_ID" l10nupdate

	lxc attach -- useradd -u "$VAGRANT_UID" -Ng "$VAGRANT_GID" -d /home/vagrant -m -s /bin/bash vagrant
	lxc attach -- useradd -u "$L10NUPDATE_ID" -Ng "$L10NUPDATE_ID" -d /home/l10nupdate -m -s /bin/bash l10nupdate
	lxc attach -- useradd -G "$L10NUPDATE_ID" -d /home/mwdeploy -m -s /bin/bash mwdeploy

	for user in root vagrant l10nupdate mwdeploy;
	do echo "$user:$PASSWORD";
	done \| lxc attach -- chpasswd

	for user in vagrant l10nupdate mwdeploy; do
	lxc attach -- sudo -su "$user" mkdir -m 0700 /home/"$user"/.ssh
	lxc attach -- sudo -u "$user" sh -c "cat > /home/${user}/.ssh/authorized_keys" \
	< /home/"$user"/.ssh/id_rsa.pub
	lxc attach -- sudo -su "$user" chmod 0600 /home/"$user"/.ssh/authorized_keys
	done

	lxc attach -q -- mkdir -p /srv/mediawiki/wmf-config
	lxc attach -q -- touch /srv/mediawiki/wmf-config/InitialiseSettings.php
	lxc attach -q -- chown -R mwdeploy:mwdeploy /srv/mediawiki

	echo 'Enabling mockbase systemd service in base container'
	lxc attach -q -- systemctl -q enable /etc/mockbase/mockbase.service

	echo 'Installing /usr/local/bin scripts and NRPE checks'
	install -o root -m 0755 -t $BASE_ROOT/usr/local/bin /vagrant/files/bin/*
	install -o root -m 0644 -t $BASE_ROOT/etc/nagios/nrpe.d /vagrant/files/nrpe.d/*.cfg

	echo 'Installing git-fat into base container'
	curl -s "$GITFAT_URL" > $BASE_ROOT/usr/local/bin/git-fat
	chmod +x $BASE_ROOT/usr/local/bin/git-fat

	echo 'Authorizing sudo for vagrant user on base container'
	lxc attach -- sh -c 'cat > /etc/sudoers.d/mockbase' < /vagrant/files/sudoers
	lxc attach -- sh -c 'cat > /etc/scap.cfg' < /vagrant/files/scap.cfg

	echo 'Install SSH private key for vagrant user'
	install -o "$VAGRANT_UID" -g "$VAGRANT_GID" -m 0400 /home/vagrant/.ssh/id_rsa $BASE_ROOT/home/vagrant/.ssh/

	echo 'Authorizing SSH host key of deploy host on base container'
	ssh-keyscan -H -t ecdsa 192.168.122.1 >> $BASE_ROOT/etc/ssh/ssh_known_hosts 2> /dev/null

	echo 'Adding global scap config'
	install -o root -m 0644 /vagrant/files/scap.cfg $BASE_ROOT/etc/

	echo 'Stopping base container'
	lxc stop
	fi

	# Clone base container and add clones to the mockbase target group
	if [ -z "$(lxc-ls $CONTAINER_PREFIX-[0-9])" ]; then
	mkdir -p /etc/dsh/group
	truncate -s 0 /etc/dsh/group/mockbase

	for i in {1..10}; do
	clone=$CONTAINER_PREFIX-$(printf '%02d' $i)

	lxc-copy -s -n "$BASE_CONTAINER" -N "$clone"

	cat >> /var/lib/lxc/"$clone"/config <<END
	lxc.start.auto = 1
	lxc.group = onboot
	lxc.mount.entry=/scap /var/lib/lxc/$clone/rootfs/srv/deployment/scap/scap ro bind 0 0
	END

	# Add clone to mockbase target group
	echo "$clone" >> /etc/dsh/group/mockbase
	done
	fi

	# Start any stopped clone containers
	mapfile -t stopped < <(lxc-ls --stopped $CONTAINER_PREFIX-[0-9])

	for container in "${stopped[@]}"; do
	echo "Starting container $container"
	lxc-start -n "$container" -d
	done

	# Wait until SSH is up on each container then authorize its host key
	for container in "${stopped[@]}"; do
	lxc_wait_for_ssh "$container"

	echo "Authorizing host key for $container"
	if [ -f /etc/ssh/ssh_known_hosts ]; then
	ssh-keygen -R "$container" -f /etc/ssh/ssh_known_hosts 2> /dev/null
	# Fix permissions after running ssh-keygen
	chmod 0644 /etc/ssh/ssh_known_hosts
	fi

	ssh-keyscan -H -t ecdsa "$container" >> /etc/ssh/ssh_known_hosts 2> /dev/null
	done

	SCAP_REPO='/srv/deployment/scap/scap'

	# Prepare deploy repo
	if [ ! -d "$SCAP_REPO" ]; then
	printf '%s\n' 'Cloning scap into the deployment directory'
	sudo -su vagrant mkdir -p "$SCAP_REPO"
	sudo -su vagrant rsync \
	--exclude=*.swp \
	--exclude=.git \
	--delete \
	-qr \
	/vagrant/scap/. "${SCAP_REPO}/."
	fi

	if [ ! -d /srv/mediawiki ]; then
	mkdir -p /srv/mediawiki
	chown -R mwdeploy:mwdeploy /srv/mediawiki
	fi

	if [ ! -f /etc/sudoers.d/scap-vagrant ]; then
	install -o root -m 0400 /vagrant/files/sudoers /etc/sudoers.d/scap-vagrant
	fi

	printf '%s\n' 'Setting up Rsync server'
	if [ ! -f /etc/rsyncd.conf ]; then
	cat > /etc/rsyncd.conf <<EOD
	uid = nobody
	gid = nogroup
	use chroot = yes

	log format = %t %a %m %f %b
	syslog facility = local3
	timeout = 300
	address = 0.0.0.0

	[ common ]
	path = /srv/mediawiki-staging
	read only = yes
	write only = no
	list = yes
	uid = 0
	gid = 0
	max connections = 0
	EOD
	fi

	service rsync start

	# Clone mw onto master
	if [ ! -d /srv/mediawiki-staging/.git ]; then
	printf 'Cloning %s to /srv/mediawiki-staging\n' "$MW_CONFIG"
	git clone --depth 1 -q "$MW_CONFIG" /srv/mediawiki-staging
	git -C /srv/mediawiki-staging fetch https://gerrit.wikimedia.org/r/operations/mediawiki-config refs/changes/00/490900/1
	git -C /srv/mediawiki-staging cherry-pick FETCH_HEAD
	chown -R vagrant:vagrant /srv/mediawiki-staging
	fi

	printf 'Faking /etc/wikimedia-cluster\n'
	echo 'eqiad' > /etc/wikimedia-cluster

	printf 'Setting up php-%s mediawiki checkout\n' "$MW_VERSION"
	if [ ! -f /srv/mediawiki/wikiversions-labs.json ]; then
	sudo -u mwdeploy -- /srv/deployment/scap/scap/bin/scap pull
	fi

	cd /srv/mediawiki-staging
	sudo -u vagrant /srv/deployment/scap/scap/bin/scap prep "$MW_VERSION"

	unlink /srv/mediawiki-staging/php
	ln -s /srv/mediawiki-staging/php-"$MW_VERSION" /srv/mediawiki-staging/php

	chown -R vagrant:vagrant /srv/mediawiki-staging
	chown -R mwdeploy:mwdeploy /srv/mediawiki

	# Setup SSH_AUTH_SOCK for scap
	printf 'Setting up SSH_AUTH_SOCK for mwdeploy\n'
	if [ ! -S /home/mwdeploy/scap.sock ]; then
	sudo -u mwdeploy -- ssh-agent -a /home/mwdeploy/scap.sock
	fi

	fingerprint=$(ssh-keygen -l -f /home/mwdeploy/.ssh/id_rsa \| cut -d ' ' -f2)

	if ! ssh-add -l \| grep -q "$fingerprint" ; then
	sudo -u mwdeploy SSH_AUTH_SOCK=/home/mwdeploy/scap.sock ssh-add /home/mwdeploy/.ssh/id_rsa
	fi

	printf 'Setting up dsh_targets for mediawiki deploy\n'
	cp /vagrant/files/mwdeploy/dsh_{proxies,masters,targets} /etc/dsh/group/

	# Fix scap config for MediaWiki
	printf 'Setting up /etc/scap.cfg mediawiki deploy\n'
	if [ ! -f /etc/scap.cfg ]; then
	install -o root -m 0644 /vagrant/files/scap.cfg /etc/scap.cfg
	fi

	# Use the example PrivateSettings.php file, to make php lint happy
	if [ ! -f /srv/mediawiki-staging/private/PrivateSettings.php ]; then
	install -o vagrant -m 774 /srv/mediawiki-staging/private/PrivateSettings.php.example \
	/srv/mediawiki-staging/private/PrivateSettings.php
	fi

	# Setup mwscript, used to rebuild the cdb files
	if [ ! -x /usr/local/bin/mwscript ]; then
	install -o root -m 755 /vagrant/files/mwscript /usr/local/bin/mwscript
	fi

	sudo -u vagrant /srv/deployment/scap/scap/bin/scap update-wikiversions all-labs "$MW_VERSION"

	if [ ! -f /srv/mediawiki/wikiversions-labs.php ]; then
	sudo -u mwdeploy -- /srv/deployment/scap/scap/bin/scap pull
	sudo -u mwdeploy -- /srv/deployment/scap/scap/bin/scap wikiversions-compile
	fi

	mkdir -p "/srv/mediawiki-staging/php-${MW_VERSION}/cache/l10n"
	chown -R l10nupdate "/srv/mediawiki-staging/php-${MW_VERSION}/cache/l10n"

Event Timeline

thcipriani created this paste.Feb 19 2019, 2:53 PM

(An Untitled Masterwork)ActivePublicActions

Event Timeline

(An Untitled Masterwork)
ActivePublic
Actions