Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F3264919
combine_es_indices.php
No One
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Authored By
EBernhardson
Jan 21 2016, 8:44 PM
2016-01-21 20:44:00 (UTC+0)
Size
1 KB
Referenced Files
None
Subscribers
None
combine_es_indices.php
View Options
<?php
$languages
=
array
();
while
(
$line
=
fgets
(
STDIN
)
)
{
$pieces
=
explode
(
' '
,
preg_replace
(
'/ +/'
,
' '
,
$line
)
);
$language
=
preg_replace
(
'/wik(.*)$/'
,
'wik'
,
$pieces
[
2
]
);
if
(
false
!==
strpos
(
$pieces
[
2
],
'apifeatureusage'
)
)
{
continue
;
}
elseif
(
false
!==
strpos
(
$pieces
[
2
],
'_content_'
)
)
{
$indexType
=
'content'
;
}
else
{
$indexType
=
'general'
;
}
$languages
[
$language
][
$indexType
][
$pieces
[
2
]]
=
array
(
'shards'
=>
$pieces
[
3
],
'replicas'
=>
$pieces
[
4
],
'size'
=>
$pieces
[
8
],
);
}
$GB
=
pow
(
2
,
30
);
$totalShards
=
0
;
$reduction
=
0
;
echo
"Checking "
,
count
(
$language
),
" languages
\n
"
;
foreach
(
$languages
as
$language
=>
$indexTypes
)
{
foreach
(
$indexTypes
as
$indexType
=>
$indices
)
{
$langShards
=
0
;
$langSize
=
0
;
$wikis
=
array
();
foreach
(
$indices
as
$index
=>
$data
)
{
$totalShards
+=
$data
[
'shards'
]
*
(
1
+
$data
[
'replicas'
]);
if
(
false
!==
strpos
(
$index
,
'titlesuggest'
)
)
{
// titlesuggest stays 1 index per wiki
continue
;
}
$langShards
+=
$data
[
'shards'
]
*
(
1
+
$data
[
'replicas'
]);
$langSize
+=
$data
[
'size'
];
$wikis
[]
=
$index
;
}
$optimalPrimaries
=
ceil
(
$langSize
/
(
2
*
$GB
)
);
if
(
$optimalPrimaries
>
1
)
{
// if more than 1 primary shards, not worth it
continue
;
}
// assumes 1 primary and 2 replicas
$optimalShards
=
$optimalPrimaries
*
3
;
if
(
$optimalShards
>=
$langShards
)
{
continue
;
}
$langSizeGB
=
number_Format
(
$langSize
/
$GB
,
2
);
echo
implode
(
', '
,
$wikis
),
" could be reduced from $langShards to $optimalShards shards for a size of $langSizeGB GB
\n
"
;
$reduction
+=
$langShards
-
$optimalShards
;
}
}
$percentDiff
=
number_format
(
100
*
$reduction
/
$totalShards
,
1
);
echo
"
\n
Could reduce shard count across cluster by $reduction ({$percentDiff}%)
\n
"
;
File Metadata
Details
Attached
Mime Type
text/plain; charset=utf-8
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3272683
Default Alt Text
combine_es_indices.php (1 KB)
Attached To
Mode
P2510 combine_es_indices.php
Attached
Detach File
Event Timeline
Log In to Comment