Page MenuHomePhabricator
Files F8691788

Makefile
awight (Adam Wight)
Actions

Authored By
awight
Jul 10 2017, 4:21 PM
Size
10 KB
Referenced Files
None
Subscribers
None

Makefile
View Options

.DELETE_ON_ERROR:
all: models
models: \
models/enwiki.draft_quality.gradient_boosting.model \
models/enwiki.draft_quality_with_sentiment.gradient_boosting.model
#datasets/enwiki.draft_quality.50k_stratified.json: \
# datasets/enwiki.draft_quality.201508-201608.tsv.bz2
# ( \
# bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | \
# tsv2json str int str int str | \
# grep '"draft_quality": "OK"' | shuf -n 26257; \
# bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | \
# tsv2json str int str int str | \
# grep -v '"draft_quality": "OK"' \
# ) > \
# datasets/enwiki.draft_quality.50k_stratified.json
datasets/enwiki.draft_quality.201508-201608.json.bz2: \
datasets/enwiki.draft_quality.201508-201608.tsv.bz2
bzcat $< | \
tsv2json str int str int str | bzip2 -c > $@
datasets/enwiki.draft_quality.201508-201608.with_text.json.bz2: \
datasets/enwiki.draft_quality.201508-201608.json.bz2
bzcat $< | \
revscoring fetch_text --host https://en.wikipedia.org \
--verbose | bzip2 -c > $@
datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2: \
datasets/enwiki.draft_quality.201508-201608.with_text.json.bz2
bzcat $< | \
wikiclass extract_from_text \
draftquality.feature_lists.enwiki.draft_quality \
draftquality.feature_lists.enwiki.draft_quality_with_sentiment \
--extractors 12 \
--verbose | bzip2 -c > $@
tuning_reports/enwiki.draft_quality.md: \
datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
bzcat $< | \
revscoring tune \
config/classifiers.params.yaml \
draftquality.feature_lists.enwiki.draft_quality \
draft_quality \
--cv-timeout=60 \
--scoring=accuracy \
--debug > $@
datasets/enwiki.draft_quality.201508-201608.10k.with_cache.json: \
datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
bzcat $< | shuf -n 10000 > $@
datasets/enwiki.draft_quality.201508-201608.test.10k.with_cache.json: \
datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
bzcat $< | shuf -n 10000 > $@
datasets/enwiki.draft_quality.201508-201608.shuffled.with_cache.json.bz2: \
datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
bzcat $< | shuf | bzip2 -c > $@
datasets/enwiki.draft_quality.201508-201608.train.700k.with_cache.json.bz2: \
datasets/enwiki.draft_quality.201508-201608.shuffled.with_cache.json.bz2
bzcat $< | tail -n +100000 | head -n 700000 | bzip2 -c > $@
datasets/enwiki.draft_quality.201508-201608.test.100k.with_cache.json.bz2: \
datasets/enwiki.draft_quality.201508-201608.shuffled.with_cache.json.bz2
bzcat $< | head -n 100000 | bzip2 -c > $@
models/enwiki.draft_quality_with_sentiment.gradient_boosting.model: \
datasets/enwiki.draft_quality.201508-201608.train.700k.with_cache.json.bz2
bzcat $< | \
revscoring train_model \
revscoring.scorer_models.GradientBoosting \
draftquality.feature_lists.enwiki.draft_quality_with_sentiment \
draft_quality \
-p 'learning_rate=0.01' \
-p 'max_features="log2"' \
-p 'max_depth=7' \
-p 'n_estimators=700' \
--version 0.0.1 > $@
models/enwiki.draft_quality.gradient_boosting.model: \
datasets/enwiki.draft_quality.201508-201608.train.700k.with_cache.json.bz2
bzcat $< | \
revscoring train_model \
revscoring.scorer_models.GradientBoosting \
draftquality.feature_lists.enwiki.draft_quality \
draft_quality \
-p 'learning_rate=0.01' \
-p 'max_features="log2"' \
-p 'max_depth=7' \
-p 'n_estimators=700' \
--version 0.0.1 > $@
tests: \
test-wo-sentiment \
test-sentiment
# NOTE test-inclusive model is left for manual pickup as "test"
test-wo-sentiment: \
datasets/enwiki.draft_quality.201508-201608.test.100k.with_cache.json.bz2 \
models/enwiki.draft_quality.gradient_boosting.model
bzcat $< | \
revscoring test_model \
models/enwiki.draft_quality.gradient_boosting.model \
draft_quality \
--observations="<stdin>" \
-s 'table' -s 'accuracy' -s 'roc' -s 'f1' \
-s 'filter_rate_at_recall(min_recall=0.75)' \
-s 'filter_rate_at_recall(min_recall=0.9)' > $@
# NOTE test-inclusive model is left for manual pickup as "test"
test-sentiment: \
datasets/enwiki.draft_quality.201508-201608.test.100k.with_cache.json.bz2 \
models/enwiki.draft_quality_with_sentiment.gradient_boosting.model
bzcat $< | \
revscoring test_model \
models/enwiki.draft_quality_with_sentiment.gradient_boosting.model \
--observations="<stdin>" \
draft_quality \
-s 'table' -s 'accuracy' -s 'roc' -s 'f1' \
-s 'filter_rate_at_recall(min_recall=0.75)' \
-s 'filter_rate_at_recall(min_recall=0.9)' > $@
############### Big dataset ###################################################
host=analytics-store.eqiad.wmnet
user=research
mysql_args = -h $(host) -u $(user)
mysqlc = mysql $(mysql_args)
mysqlc_import = mysqlimport $(mysql_args) --local
datasets/enwiki.draft_quality.201508-201608.tsv.bz2: \
datasets/enwiki.draft_quality.201508.tsv.bz2 \
datasets/enwiki.draft_quality.201509.tsv.bz2 \
datasets/enwiki.draft_quality.201510.tsv.bz2 \
datasets/enwiki.draft_quality.201511.tsv.bz2 \
datasets/enwiki.draft_quality.201512.tsv.bz2 \
datasets/enwiki.draft_quality.201601.tsv.bz2 \
datasets/enwiki.draft_quality.201602.tsv.bz2 \
datasets/enwiki.draft_quality.201603.tsv.bz2 \
datasets/enwiki.draft_quality.201604.tsv.bz2 \
datasets/enwiki.draft_quality.201605.tsv.bz2 \
datasets/enwiki.draft_quality.201606.tsv.bz2 \
datasets/enwiki.draft_quality.201607.tsv.bz2
( \
bzcat datasets/enwiki.draft_quality.201508.tsv.bz2; \
bzcat datasets/enwiki.draft_quality.201509.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201510.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201511.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201512.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201601.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201602.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201603.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201604.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201605.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201606.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201607.tsv.bz2 | tail -n+2 \
) | bzip2 -c > $@
datasets/enwiki.draft_quality.201508.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201508", @end="201509";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201509.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201509", @end="201510";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201510.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201510", @end="201511";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201511.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201511", @end="201512";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201512.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201512", @end="201601";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201601.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201601", @end="201602";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201602.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201602", @end="201603";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201603.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201603", @end="201604";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201604.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201604", @end="201605";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201605.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201605", @end="201606";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201606.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201606", @end="201607";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201607.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201607", @end="201608";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.75_not_OK_sample.tsv: \
datasets/enwiki.draft_quality.201508-201608.tsv.bz2
( \
bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | head -n 1; \
bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | grep -P 'spam$$' | shuf -n 25; \
bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | grep -P 'attack$$' | shuf -n 25; \
bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | grep -P 'vandalism$$' | shuf -n 25 \
) > $@
datasets/enwiki.draft_quality.201608-201701.tsv.bz2: \
datasets/enwiki.draft_quality.201608.tsv.bz2 \
datasets/enwiki.draft_quality.201609.tsv.bz2 \
datasets/enwiki.draft_quality.201610.tsv.bz2 \
datasets/enwiki.draft_quality.201611.tsv.bz2 \
datasets/enwiki.draft_quality.201612.tsv.bz2
( \
bzcat datasets/enwiki.draft_quality.201608.tsv.bz2; \
bzcat datasets/enwiki.draft_quality.201609.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201610.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201611.tsv.bz2 | tail -n+2; \
bzcat datasets/enwiki.draft_quality.201612.tsv.bz2 | tail -n+2 \
) | bzip2 -c > $@
datasets/enwiki.draft_quality.201608.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201608", @end="201609";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201609.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201609", @end="201610";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201610.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201610", @end="201611";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201611.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201611", @end="201612";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201612.tsv.bz2: \
sql/draft_quality.variables.sql
echo 'SET @start="201612", @end="201701";' | \
cat - $< | \
$(mysqlc) enwiki | bzip2 -c > $@
datasets/enwiki.draft_quality.201608-201701.json: \
datasets/enwiki.draft_quality.201608-201701.tsv.bz2
bzcat $< | \
tsv2json str int str int str > $@
datasets/enwiki.draft_quality.201608-201701.with_text.json.bz2: \
datasets/enwiki.draft_quality.201608-201701.json
cat $< | \
revscoring fetch_text --host https://en.wikipedia.org \
--verbose | bzip2 -c > $@
datasets/enwiki.draft_quality.201608-201701.with_cache.json.bz2: \
datasets/enwiki.draft_quality.201608-201701.with_text.json.bz2
bzcat $< | \
wikiclass extract_from_text \
draftquality.feature_lists.enwiki.draft_quality \
--verbose | bzip2 -c > $@

File Metadata

Mime Type
text/x-makefile
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
4767834
Default Alt Text
Makefile (10 KB)

Event Timeline

Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits