source: ValBot/Python/check_interwiki_links.py@ 1170

# Check Interwiki Links
# by iritscen@yahoo.com
# Looks at each link on a page (or in all the pages in a category) which uses a registered
# interwiki prefix and loads the linked page, verifying that it exists and that any section
# link, if present, is valid as well. The output will use the word "ERROR" when it cannot
# validate the interwiki link.
# Recommended viewing width:
# |---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---|

import os

from urllib.parse import urljoin

import pywikibot
import re
import requests

from pywikibot.bot import QuitKeyboardInterrupt
from pywikibot import pagegenerators
from pywikibot.tools.formatter import color_format
from pywikibot.comms.http import fetch
from pywikibot.specialbots import UploadRobot
from bs4 import BeautifulSoup

# Parallel arrays based on https://wiki.oni2.net/Special:Interwiki
interwiki_prefixes = ('acronym', 'cache', 'commons', 'dictionary', 'google', 'metawikimedia', 'mw', 'wikibooks', 'wikidata', 'wikimedia', 'wikinews', 'wikipedia', 'wikiquote', 'wikisource', 'wikispecies', 'wikiversity', 'wikivoyage', 'wikt', 'wiktionary', 'wp')

interwiki_urls = ('http://www.acronymfinder.com/~/search/af.aspx?string=exact&Acronym=', 'http://www.google.com/search?q=cache:', 'https://commons.wikimedia.org/wiki/', 'http://www.dict.org/bin/Dict?Database=*&Form=Dict1&Strategy=*&Query=', 'http://www.google.com/search?q=', 'https://meta.wikimedia.org/wiki/', 'https://www.mediawiki.org/wiki/', 'https://en.wikibooks.org/wiki/', 'https://www.wikidata.org/wiki/', 'https://foundation.wikimedia.org/wiki/', 'https://en.wikinews.org/wiki/', 'https://en.wikipedia.org/wiki/', 'https://en.wikiquote.org/wiki/', 'https://wikisource.org/wiki/', 'https://species.wikimedia.org/wiki/', 'https://en.wikiversity.org/wiki/', 'https://en.wikivoyage.org/wiki/', 'https://en.wiktionary.org/wiki/', 'https://en.wiktionary.org/wiki/', 'https://en.wikipedia.org/wiki/')

pages_checked = 0
iw_found = 0
errors_issued = 0

# Searches the given page text for interwiki links
def scan_for_iw_links(page_text):
    global pages_checked
    global iw_found
    global errors_issued
    pages_checked = pages_checked + 1
    cur = 0

    for prefix in interwiki_prefixes:
        # Isolate strings that start with "[[prefix:" and end with "|" or "]"
        iw_link = "\[\[" + prefix + ":[^|\]]*(\||\])"
        for match in re.finditer(iw_link, page_text):
            # Extract just the page title from this regex match
            s = match.start() + 2 + len(prefix) + 1
            e = match.end() - 1

            # Sometimes we used a space char. instead of a '_', so fix that before querying
            page_title = page_text[s:e].replace(' ', '_')

            # Construct full URL for the particular wiki
            iw_url = interwiki_urls[cur] + page_title
            pywikibot.stdout('   Validating {0} link "{1}"'.format(prefix, page_title))
            iw_found = iw_found + 1

            # Adjust URL if this is a foreign-language WP link
            if re.match("^[a-zA-Z]{2}:", page_title):
                lang_code = page_title[0:2] + "."
                # "wp:" is the Wikipedia: namespace, not a language
                if lang_code != "wp." and lang_code != "WP.":
                    iw_url = iw_url.replace('en.', lang_code)
                    iw_url = iw_url.replace(page_title[0:3], '')

            # Test the URL
            #pywikibot.stdout('   Testing URL "{}"'.format(iw_url))
            response = fetch(iw_url)

            # Redirects are followed automatically by fetch() and treated as "200"s, so the
            # way we tell that a redirect occurred is by checking the history
            if response.history != []:
                pywikibot.stdout('   ERROR: Got redirection code ({0}) on URL "{1}".'.format(response.history[0], iw_url))
                errors_issued = errors_issued + 1
            elif response.status_code != 200:
                pywikibot.stdout('   ERROR: Got response code {0} on URL "{1}".'.format(response.status_code, iw_url))
                errors_issued = errors_issued + 1
            elif '#' in page_title:
                # Isolate section link
                page_name, anchor_name = page_title.split('#')
                
                # Convert dot-notation hex entities to proper characters
                anchor_name = anchor_name.replace('.22', '"')
                anchor_name = anchor_name.replace('.27', '\'')
                anchor_name = anchor_name.replace('.28', '(')
                anchor_name = anchor_name.replace('.29', ')')
                
                # Read linked page to see if it really has this anchor link
                soup = BeautifulSoup(response.text, 'html.parser')
                found_section = False
                for span_tag in soup.findAll('span'):
                    span_name = span_tag.get('id', None)
                    if span_name == anchor_name:
                        #pywikibot.stdout('Found section!')
                        found_section = True
                        break
                if found_section == False:
                    pywikibot.stdout('   ERROR: Could not find section {0} on page {1}.'.format(anchor_name, page_name))
                    errors_issued = errors_issued + 1
        cur = cur + 1

def main(*args):
    cat_name = ''
    page_name = ''

    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()

    for arg in local_args:
        if arg.startswith('-cat:'):
            cat_name = arg[5:]
        elif arg.startswith('-page:'):
            page_name = arg[6:]

    site = pywikibot.Site()

    #pywikibot.stdout('The members of the requests.models.Response class are:')
    #pywikibot.stdout(format(dir(requests.models.Response)))

    if cat_name != '':
        cat_obj = pywikibot.Category(site, cat_name)
        generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True)
        for page in pagegenerators.PreloadingGenerator(generator, 100):
            pywikibot.stdout('Checking page "{}"'.format(page.title()))
            scan_for_iw_links(page.text)
    elif page_name != '':
        page = pywikibot.Page(site, page_name)
        pywikibot.stdout('Checking page "{}"'.format(page.title()))
        scan_for_iw_links(page.text)

    global pages_checked
    global iw_found
    global errors_issued

    page_str = "pages"
    if pages_checked == 1:
        page_str = "page"

    link_str = "links"
    if iw_found == 1:
        link_str = "link"

    pywikibot.stdout('Checked {0} {1} and found {2} interwiki {3}.'.format(pages_checked, page_str, iw_found, link_str))

    error_str = "errors were"
    if errors_issued == 1:
        error_str = "error was"

    pywikibot.stdout('{0} {1} encountered in validating these links.'.format(errors_issued, error_str))

if __name__ == '__main__':
    main()