source: ValBot/Python/check_interwiki_links.py@ 1169

import os

from urllib.parse import urljoin

import pywikibot
import re

from pywikibot.bot import QuitKeyboardInterrupt
from pywikibot import pagegenerators
from pywikibot.tools.formatter import color_format
from pywikibot.comms.http import fetch
from pywikibot.specialbots import UploadRobot
from bs4 import BeautifulSoup

# Parallel arrays based on https://wiki.oni2.net/Special:Interwiki
interwiki_prefixes = ('acronym', 'cache', 'commons', 'dictionary', 'google', 'metawikimedia', 'mw', 'wikibooks', 'wikidata', 'wikimedia', 'wikinews', 'wikipedia', 'wikiquote', 'wikisource', 'wikispecies', 'wikiversity', 'wikivoyage', 'wikt', 'wiktionary', 'wp')

interwiki_urls = ('http://www.acronymfinder.com/~/search/af.aspx?string=exact&Acronym=', 'http://www.google.com/search?q=cache:', 'https://commons.wikimedia.org/wiki/', 'http://www.dict.org/bin/Dict?Database=*&Form=Dict1&Strategy=*&Query=', 'http://www.google.com/search?q=', 'https://meta.wikimedia.org/wiki/', 'https://www.mediawiki.org/wiki/', 'https://en.wikibooks.org/wiki/', 'https://www.wikidata.org/wiki/', 'https://foundation.wikimedia.org/wiki/', 'https://en.wikinews.org/wiki/', 'https://en.wikipedia.org/wiki/', 'https://en.wikiquote.org/wiki/', 'https://wikisource.org/wiki/', 'https://species.wikimedia.org/wiki/', 'https://en.wikiversity.org/wiki/', 'https://en.wikivoyage.org/wiki/', 'https://en.wiktionary.org/wiki/', 'https://en.wiktionary.org/wiki/', 'https://en.wikipedia.org/wiki/')

pages_checked = 0
iw_found = 0
problems_found = 0

# Searches the given page text for interwiki links
def scan_for_iw_links(page_text):
    global pages_checked
    global iw_found
    global problems_found
    pages_checked = pages_checked + 1
    cur = 0

    for prefix in interwiki_prefixes:
        # Isolate strings that start with "[[prefix:" and end with "|" or "]"
        iw_link = "\[\[" + prefix + ":[^|\]]*(\||\])"
        for match in re.finditer(iw_link, page_text):
            # Extract just the page title from this regex match
            s = match.start() + 2 + len(prefix) + 1
            e = match.end() - 1

            # Sometimes we used a space char. instead of a '_', so fix that before querying
            page_title = page_text[s:e].replace(' ', '_')

            # Construct full URL for the particular wiki
            iw_url = interwiki_urls[cur] + page_title
            pywikibot.output('Found {0} link {1}.'.format(prefix, page_title))
            iw_found = iw_found + 1

            # Adjust URL if this is a foreign-language WP link
            if re.match("^[a-zA-Z]{2}:", page_title):
                lang_code = page_title[0:2] + "."
                # "wp:" is the Wikipedia: namespace, not a language
                if lang_code != "wp." and lang_code != "WP.":
                    iw_url = iw_url.replace('en.', lang_code)
                    iw_url = iw_url.replace(page_title[0:3], '')

            # Test the URL
            #pywikibot.output('Testing URL {}...'.format(iw_url))
            response = fetch(iw_url)

            # Redirects are followed automatically by fetch() and treated as "200"s, so the
            # way we tell that a redirect occurred is by checking the history
            if response.history != []:
                pywikibot.output('WARNING: Redirected from {}.'.format(response.history))
                problems_found = problems_found + 1
            elif response.status_code != 200:
                #pywikibot.output('WARNING: Got response code {}.'.format(response.status_code)) # commented out because fetch() already prints such a msg
                problems_found = problems_found + 1
            elif '#' in page_title:
                # Isolate section link
                pywikibot.output('Detected section link on page {0}.'.format(page_title))
                page_name, anchor_name = page_title.split('#')
                
                # Convert dot-notation hex entities to proper characters
                anchor_name = anchor_name.replace('.22', '"')
                anchor_name = anchor_name.replace('.27', '\'')
                anchor_name = anchor_name.replace('.28', '(')
                anchor_name = anchor_name.replace('.29', ')')
                
                # Read linked page to see if it really has this anchor link
                soup = BeautifulSoup(response.text, 'html.parser')
                found_section = False
                for tag in soup.findAll('a'):
                    link = tag.get('href', None)
                    if not link:
                        #pywikibot.output('It is not a link.')
                        continue
                    #pywikibot.output('Got link {0}.'.format(link))
                    if not link.startswith('#'):
                        continue
                        
                    if link == '#' + anchor_name:
                        pywikibot.output('Found section link!')
                        found_section = True
                        break
                if found_section == False:
                    pywikibot.output('Could not find section {0} on page {1}.'.format(anchor_name, page_name))
                    problems_found = problems_found + 1
        cur = cur + 1

def main(*args):
    cat_name = ''
    page_name = ''

    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()

    for arg in local_args:
        if arg.startswith('-cat:'):
            cat_name = arg[5:]
        elif arg.startswith('-page:'):
            page_name = arg[6:]

    site = pywikibot.Site()

    # This line of code enumerates the methods in the 'page' class
    #pywikibot.stdout(format(dir(page)))

    if cat_name != '':
        cat_obj = pywikibot.Category(site, cat_name)
        generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True)
        for page in pagegenerators.PreloadingGenerator(generator, 100):
            pywikibot.stdout('Checking page {0}'.format(page.title()))
            scan_for_iw_links(page.text)
    elif page_name != '':
        page = pywikibot.Page(site, page_name)
        pywikibot.stdout('Checking page {0}'.format(page.title()))
        scan_for_iw_links(page.text)

    global pages_checked
    global iw_found
    global problems_found
    pywikibot.stdout('Checked {0} page(s) and found {1} interwiki link(s) with {2} problem(s).'.format(pages_checked, iw_found, problems_found))

if __name__ == '__main__':
    main()