Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Paste
P8448
pccvtc.py
Active
Public
Actions
Authored by
•
ema
on Apr 27 2019, 12:12 PM.
Edit Paste
Archive Paste
View Raw File
Subscribe
Mute Notifications
Award Token
Flag For Later
Tags
None
Referenced Files
F28808008: raw.txt
Apr 27 2019, 12:12 PM
2019-04-27 12:12:31 (UTC+0)
Subscribers
None
#!/usr/bin/env python3
import
os
import
re
import
sys
import
requests
DC
=
(
'eqiad'
,
'codfw'
,
'esams'
,
'ulsfo'
,
'eqsin'
)
CLUSTERS
=
(
'text'
,
'upload'
)
PATH_RE
=
re
.
compile
(
'^(/etc/varnish|/usr/share/varnish/)'
)
COMPILER_RE
=
re
.
compile
(
'.*(https://puppet-compiler.wmflabs.org/compiler[0-9]{4}/[0-9]+/)'
)
PCC
=
'../../../../utils/pcc'
def
find_cluster
(
hostname
):
# eg: cp4021.ulsfo.wmnet -> DC[3] -> 'ulsfo'
idx
=
int
(
hostname
[
2
])
-
1
dc
=
DC
[
idx
]
for
cluster
in
CLUSTERS
:
url
=
'https://config-master.wikimedia.org/pybal/{}/{}'
.
format
(
dc
,
cluster
)
r
=
requests
.
get
(
url
)
if
hostname
in
r
.
text
:
return
cluster
raise
Exception
(
'Unknown cluster for {}'
.
format
(
hostname
))
def
get_pcc_url
(
hostname
,
patch_id
):
cmd
=
' '
.
join
((
PCC
,
patch_id
,
hostname
))
for
line
in
os
.
popen
(
cmd
)
.
readlines
():
match
=
COMPILER_RE
.
match
(
line
)
if
match
:
return
match
.
group
(
1
)
raise
Exception
(
'Issues with get_pcc_url()'
)
def
dump_files
(
url
,
hostname
):
catalog_url
=
'{}/{}/change.{}.pson'
.
format
(
url
,
hostname
,
hostname
)
print
(
'
\t
Catalog URL: {}'
.
format
(
catalog_url
))
catalog
=
requests
.
get
(
catalog_url
)
.
json
()
for
resource
in
catalog
[
'resources'
]:
if
resource
[
'type'
]
!=
'File'
:
continue
if
PATH_RE
.
match
(
resource
[
'title'
])
is
None
:
continue
if
'content'
not
in
resource
[
'parameters'
]:
continue
path
=
resource
[
'title'
]
.
lstrip
(
'/'
)
os
.
makedirs
(
os
.
path
.
dirname
(
path
),
exist_ok
=
True
)
with
open
(
path
,
'wb'
)
as
f
:
print
(
'
\t
Creating {}'
.
format
(
path
))
f
.
write
(
resource
[
'parameters'
][
'content'
]
.
encode
(
'utf-8'
))
def
main
(
hostname
,
patch_id
=
'HEAD'
):
print
(
'[*] running Puppet Catalog Compiler for change {}...'
.
format
(
patch_id
))
pcc_url
=
get_pcc_url
(
hostname
,
patch_id
)
print
(
'
\t
PCC URL: {}'
.
format
(
pcc_url
))
print
(
'[*] Dumping files...'
)
dump_files
(
pcc_url
,
hostname
)
print
(
'[*] Finding cluster...'
)
cluster
=
find_cluster
(
hostname
)
print
(
'
\t
{} is a cache_{} host'
.
format
(
cluster
))
print
(
'[*] Running varnishtest...'
)
cmd
=
'sudo varnishtest {}/*.vtc'
.
format
(
cluster
)
print
(
cmd
)
if
__name__
==
'__main__'
:
if
len
(
sys
.
argv
)
!=
3
:
print
(
'Usage: {} hostname patch_id'
.
format
(
sys
.
argv
[
0
]))
sys
.
exit
(
1
)
main
(
sys
.
argv
[
1
],
sys
.
argv
[
2
])
Event Timeline
•
ema
created this paste.
Apr 27 2019, 12:12 PM
2019-04-27 12:12:31 (UTC+0)
Log In to Comment