Page MenuHomePhabricator
Paste P35447

T292270 gadgets gadgets code code!
ActivePublic

Authored by Aklapper on Oct 12 2022, 4:08 PM.
Project Tags
None
Referenced Files
F35564788: T292270 gadgets gadgets code code!
Oct 12 2022, 4:08 PM
Subscribers
# based on https://paws-public.wmflabs.org/paws-public/User:Harej_(WMF)/Lists%20of%20gadgets%20for%20every%20Wikimedia%20wiki.ipynb
# based on https://public.paws.wmcloud.org/User:SSethi_(WMF)/List%20of%20most%20used%20gadgets%20across%20wikimedia%20wikis.ipynb
# if this was my code (I am Andre Klapper, <aklapper@wikimedia.org>) this was licensed under Creative Commons Zero.
import requests
import re
import json
from collections import OrderedDict, Counter
# Download the Site Matrix to get list of Wikimedia wikis
sitematrix = 'https://en.wikipedia.org/w/api.php?action=sitematrix&format=json'
r = requests.get(sitematrix)
r = r.json()
r = r['sitematrix']
site_urls = []
for blob in r.values():
if type(blob) is dict:
for subblob in blob['site']:
if 'closed' not in subblob: # Exclude closed sites
site_urls.append(subblob['url'])
elif type(blob) is list: # "Special" wikis, including Commons and Wikidata, are different for some reason
for subblob in blob:
if 'closed' not in subblob: # Exclude closed sites
site_urls.append(subblob['url'])
# Download [[MediaWiki:Gadgets-definition]] from each of these wikis
api_request_gadgets_definition = ('/w/api.php?action=parse&format=json&prop=wikitext&page=MediaWiki%3AGadgets-definition')
gadget_defs = {}
for url in site_urls:
q = url + api_request_gadgets_definition
r = requests.get(q)
r = r.json()
if 'error' not in r: # Check if [[MediaWiki:Gadgets-definition]] exists on that wiki
gadget_defs[url] = r['parse']['wikitext']['*']
print('✅ ' + url + '/wiki/MediaWiki:Gadgets-definition')
else:
print(' ❎ ' + url + '/wiki/MediaWiki:Gadgets-definition')
print('\nGadget definition pages on active wikis: ' + str(len(gadget_defs)) + '\n')
# Data cleanup and retrieving editor information
gadget_entries = {}
for site, blob in gadget_defs.items():
# print(site)
blob = blob.split('\n')
for entry in blob:
if len(entry) > 0:
if entry[0] == '*' and '|' in entry:
gadget = entry.replace('*', '').strip()
canonical = gadget.upper().\
replace('-', '').\
replace('_', '').\
replace(' ', '').\
split('|')[0].\
split('[')[0].\
strip()
if canonical in gadget_entries:
if gadget in gadget_entries[canonical]:
gadget_entries[canonical][gadget][site] = {}
else:
gadget_entries[canonical][gadget] = {site: {}}
else:
gadget_entries[canonical] = {gadget: {site: {}}}
gadget_pages = gadget
gadget_pages = re.sub(r'\*+', '', gadget_pages)
gadget_pages = re.sub(r'\[.*?\]', '', gadget_pages)
gadget_pages = re.sub(r'<!--.*?-->', '', gadget_pages) # Must be first
gadget_pages = re.sub(r'<!--.*$', '', gadget_pages) # Must be after
gadget_pages = re.sub(r'^.*-->', '', gadget_pages) # Must be after
gadget_pages = gadget_pages.split('|')
gadget_pages = ['MediaWiki:Gadget-' + x.strip().replace(' ', '_') for x in gadget_pages \
if x.strip() != '']
for gadget_page in gadget_pages:
print(site + '/wiki/' + gadget_page)
api_request_gadget_definition = ('/w/api.php?action=parse&format=json&prop=wikitext&page=' + gadget_page)
q = site + api_request_gadget_definition
r = requests.get(q)
r = r.json()
if 'error' in r:
# Check for values with an equal sign as they imply a typo (incorrect ResourceLoader array parameters)
match1 = re.search('=', gadget_page)
if match1:
print('ERROR: Gadget definition for ' + gadget_page + ' includes an equal sign, potential typo on ' + site)
else:
# Check if listed pages actually exist but exclude description/translation page URLs that don't end in .js or .css
match2 = re.search('\.js\Z', gadget_page)
match3 = re.search('\.css\Z', gadget_page)
if (match2 or match3):
print('ERROR: Non-existing gadget page on ' + site + '/wiki/MediaWiki:Gadgets-definition : ' + gadget_page)
gadget_entries[canonical][gadget][site][gadget_page] = site
gadget_entries = OrderedDict(sorted(gadget_entries.items()))
print('\nDone')
for canonical, gadgetblob in gadget_entries.items():
counter=0
for gadget, siteblob in gadgetblob.items():
for site in siteblob.keys():
counter=counter+1
if(counter>10):
print('== ' + canonical.lower() + ' ==\n')
print(gadget)
print(counter)
# for gadget, siteblob in gadgetblob.items():
# for site in siteblob.keys():
# print(site)

Event Timeline

I think the https://en.wikipedia.org/w/api.php?action=help&modules=query%2Bgadgets Action API would remove the need for all of this home grown wikitext parsing and checking to see if pages exist while still giving the info needed to build the table. The 'id' prop gives stable name for the gadget. The 'metadata' prop includes a "category" to group things in the table and "module" information on where the source pages for the gadget live. The 'desc' prop is a HTML description of the gadget.

$ curl -qs 'https://en.wikipedia.org/w/api.php?action=query&format=json&list=gadgets&formatversion=latest&gaprop=id|metadata|desc' | jq
{
  "batchcomplete": true,
  "query": {
    "gadgets": [
      {
        "id": "modrollback",
        "metadata": {
          "settings": {
            "rights": [
              "rollback"
            ],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "browsing",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-modrollback.js"
              "mediawiki.util"
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "<sup><abbr title=\"Enabled for everyone by default.\">(D)</abbr></sup> Require confirmation before performing rollback on mobile devices"
      },
      {
        "id": "removeAccessKeys",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "browsing",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-removeAccessKeys.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [
              "user",
              "mediawiki.util"
            ],
            "peers": [],
            "messages": []
slation popup for the selected text or the word under the cursor when pushing the shift button"
      },
      {
        "id": "ImageAnnotator",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "browsing",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-ImageAnnotator.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [],
            "peers": [],
            "messages": []
          }
        },
        "desc": "<a href=\"https://commons.wikimedia.org/wiki/Help:Gadget-ImageAnnotator\" class=\"extiw\" title=\"commons:Help:Gadget-ImageAnnotator\">ImageAnnotator</a>: view image notes and comments on file description pages"
      },
      {
        "id": "imagelinks",
        "metadata": {
          "settings": {
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "browsing",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-popups.js"
            ],
            "styles": [
              "MediaWiki:Gadget-navpop.css"
            ],
            "datas": [],
            "dependencies": [
              "mediawiki.api",
              "mediawiki.user",
              "mediawiki.util",
              "user.options",
              "mediawiki.jqueryMsg"
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "<a href=\"/wiki/Wikipedia:Tools/Navigation_popups\" title=\"Wikipedia:Tools/Navigation popups\">Navigation popups</a>: article previews and editing functions pop up when hovering over links"
      },
      {
        "id": "exlinks",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-search-new-tab.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Open search results in a new tab or window when holding down the Ctrl key"
      },
      {
        "id": "PrintOptions",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "browsing",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-revisionjumper.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [],
            "peers": [],
            "messages": []
          }
        },
        "desc": "<sup><abbr title=\"Loads code managed on another WMF project\">(S)</abbr></sup> <a href=\"/wiki/User:DerHexer/revisionjumper\" title=\"User:DerHexer/revisionjumper\">revisionjumper</a>: quickly navigate between page revisions"
      },
      {
        "id": "Twinkle",
        "metadata": {
          "settings": {
            "rights": [
              "autoconfirmed"
            ],
            "skins": [],
            "actions": [],
            "default": false,
            "styles": [
              "MediaWiki:Gadget-Twinkle.css"
            ],
            "datas": [],
            "dependencies": [
              "ext.gadget.morebits",
              "ext.gadget.select2",
              "mediawiki.api",
              "mediawiki.language"
            ],
            "peers": [
              "Twinkle-pagestyles"
            ],
            "messages": []
          }
        },
        "desc": "<a href=\"/wiki/Wikipedia:Twinkle\" title=\"Wikipedia:Twinkle\">Twinkle</a>: add menu buttons to automate common tasks, including: reporting vandalism, warning vandals, requesting deletion or protection, welcoming users, and tagging articles (<a href=\"/wiki/Wikipedia:Twinkle/Preferences\" title=\"Wikipedia:Twinkle/Preferences\">preferences</a>)"
      },
      {
        "id": "morebits",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": true,
            "package": false,
            "shared": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "editing",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-legacyToolbar.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [],
            "peers": [],
            "messages": []
          }
        },
        "desc": "<a href=\"/wiki/Wikipedia:Legacy_toolbar\" title=\"Wikipedia:Legacy toolbar\">Enable the legacy (2006) editing toolbar</a>. This will be overridden by the \"Enable the editing toolbar\" option in the Editing tab."
      },
      {
        "id": "extra-toolbar-buttons",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": true,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "editing",
            "legacyscripts": false,
            "supportsUrlLoad": false
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [],
            "styles": [
              "MediaWiki:Gadget-UTCLiveClock-pagestyles.css"
            ],
            "datas": [],
            "dependencies": [],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Components for the UTCLiveClock gadget"
      },
      {
        "id": "purgetab",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "appearance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-purgetab.js"
            ],
            "styles": [],
            "datas": [],
            "datas": [],
            "dependencies": [
              "mediawiki.cookie"
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Allow toggling between tabs and dropdown menus (<a href=\"/wiki/User_talk:Edokter/MenuTabsToggle.js\" title=\"User talk:Edokter/MenuTabsToggle.js\">bug reports</a>)"
      },
      {
        "id": "dropdown-menus",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "appearance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-MoreMenu.enwiki.js",
              "MediaWiki:Gadget-dropdown-menus.js"
            ],
            "styles": [],
            "datas": [],
            ],
            "peers": [
              "dark-mode-toggle-pagestyles"
            ],
            "messages": []
          }
        },
        "desc": "<a href=\"/wiki/Wikipedia:Dark_mode_(gadget)\" title=\"Wikipedia:Dark mode (gadget)\">Dark mode</a> toggle: Enable a toggle for using a light text on dark background color scheme"
      },
      {
        "id": "dark-mode-toggle-pagestyles",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [
              "vector",
              "vector-2022",
              "minerva",
              "monobook"
            ],
            "actions": [],
            "default": false,
            "hidden": true,
            "package": false,
            "shared": false,
            "category": "appearance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [],
            "styles": [
              "MediaWiki:Gadget-dark-mode-toggle-pagestyles.css"
            ],
            "datas": [],
            "dependencies": [],
            "category": "appearance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-markblocked.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [
              "mediawiki.util",
              "mediawiki.page.ready"
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Strike out usernames that have been blocked"
      },
      {
        "id": "responsiveContent",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [
              "vector",
              "vector-2022"
            ],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-responsiveContentTimeless.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [],
            "peers": [
              "responsiveContentBaseTimeless"
            ],
            "messages": []
          }
        },
        "desc": "<sup><abbr title=\"Enabled for everyone by default.\">(D)</abbr></sup> Improved appearance for mobile and narrow screens"
      },
      {
        "id": "HideInterwikiSearchResults",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "appearance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Allow /16, /24 and /27 &#8211; /32 <a href=\"/wiki/CIDR\" class=\"mw-redirect\" title=\"CIDR\">CIDR</a> ranges on <a href=\"/wiki/Special:Contributions\" title=\"Special:Contributions\">Special:Contributions</a> forms, as well as wildcard prefix searches (e.g., \"Splark*\") (<a href=\"/wiki/MediaWiki_talk:Gadget-contribsrange.js\" title=\"MediaWiki talk:Gadget-contribsrange.js\">report issues</a>)"
      },
      {
        "id": "BugStatusUpdate",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "advanced",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-BugStatusUpdate.js"
            ],
            "styles": [],
          }
        },
        "desc": "<sup><abbr title=\"Loads code managed on another WMF project\">(S)</abbr></sup> <a href=\"https://meta.wikimedia.org/wiki/User:Krinkle/RTRC-docs\" class=\"extiw\" title=\"m:User:Krinkle/RTRC-docs\">Real-Time Recent Changes</a>: patrol recent changes in real-time (unpatrolled filter only works for new pages)"
      },
      {
        "id": "script-installer",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "advanced",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-script-installer.js"
            ],
            "styles": [],
            "datas": [],
            "dependencies": [
              "mediawiki.ui.button"
            ],
            "peers": [],
            "messages": []
          }
          }
        },
        "desc": "<a href=\"/wiki/Wikipedia:XFDC\" class=\"mw-redirect\" title=\"Wikipedia:XFDC\">XFDcloser</a>: close deletion discussions at XfD venues, including actions to implement closes"
      },
      {
        "id": "XFDcloser-core",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": true,
            "package": false,
            "shared": false,
            "category": "maintenance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
          "module": {
            "scripts": [
              "MediaWiki:Gadget-XFDcloser-core.js"
            ],
            "styles": [
              "MediaWiki:Gadget-XFDcloser-core.css"
            ],
            "datas": [],
            "dependencies": [
              "mediawiki.util",
            ],
            "datas": [],
            "dependencies": [
              "mediawiki.util",
              "mediawiki.api",
              "mediawiki.Title",
              "oojs-ui-core",
              "oojs-ui-widgets",
              "oojs-ui-windows",
              "oojs-ui.styles.icons-interactions",
              "oojs-ui.styles.icons-content",
              "oojs-ui.styles.icons-moderation",
              "oojs-ui.styles.icons-alerts"
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Beta testing components for XFDcloser gadget"
      },
      {
        "id": "libExtraUtil",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [],
            "actions": [],
            "default": false,
            "hidden": true,
            "package": false,
            "shared": false,
            "category": "maintenance",
            "legacyscripts": false,
            "supportsUrlLoad": false
          },
            ],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Utility gadget. You should not be able to see this from Special:Preferences."
      },
      {
        "id": "dark-mode",
        "metadata": {
          "settings": {
            "rights": [],
            "skins": [
              "vector",
              "vector-2022",
              "monobook",
              "modern",
              "minerva",
              "timeless"
            ],
            "actions": [],
            "default": false,
            "hidden": false,
            "package": false,
            "shared": false,
            "category": "modules",
            "legacyscripts": false,
            "supportsUrlLoad": true
          },
          "module": {
            "scripts": [],
            "styles": [
              "MediaWiki:Gadget-dark-mode.css"
            ],
            "datas": [],
            "dependencies": [],
            "peers": [],
            "messages": []
          }
        },
        "desc": "Core styling for dark mode gadget (Use the <a href=\"#mw-input-wpgadget-dark-mode-toggle\">dark mode toggle gadget above</a> to control this feature)"
      }
    ]
  }
}

@bd808 Hah, you are correct (and I even used that Action API aspect in a different script so I could have known, meh).

@bd808 Hah, you are correct (and I even used that Action API aspect in a different script so I could have known, meh).

I only learned about the api when I wrote a phab ticket about doing some screen scraping very similar this script and some helpful community folks pointed out that there was an api that would probably work better, so no shaming intended by me. :)