Page MenuHomePhabricator
Paste P54284

WDQS Graph Split Manual Data Load Notes

Authored by RKemper on Dec 7 2023, 8:12 PM.
# Downtime host(s) to reduce noise
ryankemper@cumin1001:~$ sudo -E cookbook sre.hosts.downtime --days 7 -r 'graph split experiments T350106' wdqs102[2-4].eqiad.wmnet
# Set permissions on files if not already sufficient
chmod 555 /srv/T350106/gzips/gzips/gzips/nt_wd_schol/*
# Run from `/srv/T350106/gzips/gzips/gzips/nt_wd_schol/*` to change file ext from .txt.gz to .ttl.gz
for FILE in *; do NEW_FILE="$(echo $FILE | sed 's~.txt.gz~.ttl.gz~';)"; sudo mv $FILE $NEW_FILE; done
# Disable puppet, stop blazegraph, clear out jnl file, start blazegraph, restart exporter
sudo disable-puppet "T350106" && sudo systemctl stop wdqs-blazegraph && sleep 5 && rm -fv /srv/wdqs/wikidata.jnl && sleep 5 && sudo systemctl start wdqs-blazegraph && sudo systemctl restart prometheus-blazegraph-exporter-wdqs-blazegraph.service
# Get slightly-modified into place if not present
scp ryankemper@wdqs1023.eqiad.wmnet:/home/ryankemper/
# Modify further to match file format of /srv/T350106/gzips/gzips/gzips/nt_wd_schol/* (in this case) if necessary
vi /srv/T350106/
# Run on first chunk
sudo /srv/T350106/ -n wdq -d /srv/T350106/gzips/gzips/gzips/nt_wd_schol -s 0 -e 0
# Run on all remaining chunks
sudo /srv/T350106/ -n wdq -d /srv/T350106/gzips/gzips/gzips/nt_wd_schol