popular_edge_ngrams = (
df_dump
.select(
'title',
F.split(F.col('title'), ' ').alias('title_words')
)
.select(
'title',
F.array(
F.col('title_words')[0],
F.concat_ws(' ', F.col('title_words')[0], F.col('title_words')[1]),
F.concat_ws(' ', F.col('title_words')[0], F.col('title_words')[1], F.col('title_words')[2]),
).alias('edge_ngrams')
)
.select(F.explode('edge_ngrams').alias('edge_ngram'))
.groupBy(F.col('edge_ngram'))
.agg(F.sum(F.lit(1)).alias('count'))
.orderBy(F.col('count').desc())
.limit(50)
).toPandas()
| | edge_ngram | count |
|---:|:----------------------------------|--------:|
| 0 | Articles | 582037 |
| 1 | Articles for | 565778 |
| 2 | The | 411166 |
| 3 | WikiProject | 389810 |
| 4 | List | 277978 |
| 5 | List of | 277722 |
| 6 | John | 126430 |
| 7 | People | 115276 |
| 8 | ClueBot | 106635 |
| 9 | People from | 94228 |
| 10 | Sockpuppet | 78676 |
| 11 | ClueBot III/Detailed | 76714 |
| 12 | Did | 73317 |
| 13 | Did you | 73049 |
| 14 | Did you know | 71976 |
| 15 | William | 68711 |
| 16 | Wikipedia | 66782 |
| 17 | User | 61674 |
| 18 | National | 54259 |
| 19 | Featured | 53957 |
| 20 | James | 53183 |
| 21 | David | 51824 |
| 22 | New | 49562 |
| 23 | Robert | 48416 |
| 24 | George | 47421 |
| 25 | Films | 46579 |
| 26 | American | 41888 |
| 27 | Reference | 41878 |
| 28 | A | 41754 |
| 29 | United | 41674 |
| 30 | Thomas | 40225 |
| 31 | Charles | 39088 |
| 32 | ClueBot III/Detailed Indices/User | 37545 |
| 33 | Michael | 36807 |
| 34 | South | 34350 |
| 35 | Miscellany | 33782 |
| 36 | Miscellany for | 33764 |
| 37 | Peter | 32864 |
| 38 | History | 31994 |
| 39 | Richard | 31882 |
| 40 | Paul | 30929 |
| 41 | Railway | 30622 |
| 42 | History of | 29702 |
| 43 | Suspected | 29273 |
| 44 | Buildings | 27794 |
| 45 | I | 27491 |
| 46 | Buildings and | 27382 |
| 47 | Buildings and structures | 27348 |
| 48 | 2019 | 27250 |
| 49 | 2022 | 27203 |