# Check Intrawiki Section Links # by iritscen@yahoo.com # Looks at each wikilink on a page (or in all the pages in a category) for a section link ('#'), # and loads the linked page and verifies that the named section actually exists. It also # understands section links generated through a call to Template:SectionLink. # Recommended viewing width: # |---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- --| import os from urllib.parse import urljoin import pywikibot import re from pywikibot.bot import QuitKeyboardInterrupt from pywikibot import pagegenerators from pywikibot.tools.formatter import color_format from pywikibot.comms.http import fetch from pywikibot.specialbots import UploadRobot from bs4 import BeautifulSoup # Tuple of OniGalore's namespaces intrawiki_prefixes = ('Image', 'Special', 'Talk', 'User', 'User_talk', 'OniGalore', 'OniGalore_talk', 'File', 'File_talk', 'MediaWiki', 'MediaWiki_talk', 'Template', 'Template_talk', 'Help', 'Help_talk', 'Category', 'Category_talk', 'BSL', 'BSL_talk', 'OBD', 'OBD_talk', 'AE', 'AE_talk', 'Oni2', 'Oni2_talk', 'XML', 'XML_talk') # URL for main namespace of our wiki onigalore_url = 'https://wiki.oni2.net/' # Tuple of interwiki prefixes, for passing over such links interwiki_prefixes = ('acronym', 'cache', 'commons', 'dictionary', 'google', 'metawikimedia', 'mw', 'wikibooks', 'wikidata', 'wikimedia', 'wikinews', 'wikipedia', 'wikiquote', 'wikisource', 'wikispecies', 'wikiversity', 'wikivoyage', 'wikt', 'wiktionary', 'wp') # List of chapter names, for substitution into links that use "{{Cn}}" transclusion chapter_names = ['CHAPTER_00_._COMBAT_TRAINING', 'CHAPTER_01_._TRIAL_RUN', 'CHAPTER_02_._ENGINES_OF_EVIL', 'CHAPTER_03_._PUZZLE_PIECES', 'CHAPTER_04_._TIGER_BY_THE_TAIL', 'CHAPTER_05_._HOT_PURSUIT', 'CHAPTER_06_._COUNTERATTACK', 'CHAPTER_07_._A_FRIEND_IN_NEED', 'CHAPTER_08_._AN_INNOCENT_LIFE', 'CHAPTER_09_._TRUTH_AND_CONSEQUENCES', 'CHAPTER_10_._CAT_AND_MOUSE', 'CHAPTER_11_._DREAM_DIVER', 'CHAPTER_12_._SINS_OF_THE_FATHER', 'CHAPTER_13_._PHOENIX_RISING', 'CHAPTER_14_._DAWN_OF_THE_CHRYSALIS'] # Tuple of patterns for recognizing wikilinks # Pattern 1: Detect "[[anything]]", "[[any:thing]]", "[[any|thing]]", "[[any:thi|ng]]" # Pattern 2: Detect "{{SectionLink|Page|Section name}}", "{{SectionLink||Section name}}" link_patterns = ("\[\[[^|\]]*(\||\])", "\{\{SectionLink\|[^|\}]*\|[^|\}]*\}\}") # Initialize globals debug = 0 pages_checked = 0 iw_found = 0 advice_issued = 0 errors_issued = 0 # Searches the given page text for intrawiki links with section links in them def scan_for_intrawiki_links(page_text, page_name): global debug global pages_checked global iw_found global advice_issued global errors_issued pages_checked += 1 name_printed = 0 for i, the_pattern in enumerate(link_patterns): if debug: if i == 0: pywikibot.stdout(' Checking page for wikilinks with section names.') elif i == 1: pywikibot.stdout(' Checking page for {{SectionLink}} calls.') for match in re.finditer(the_pattern, page_text): found_iw_match = False iw_url = "" page_name2 = page_name # Cut out the matched text from the page, isolating just the page+section name target_start = 2 # "[[" target_end = 1 # "|" or "]" (we only match the first ending bracket) if i == 1: target_start = 14 # "{{SectionLink|" target_end = 2 # "}}" s = match.start() + target_start # remove the link-opening markup e = match.end() - target_end # remove the link-ending markup link_text = page_text[s:e] # The second link type will look like "Page|Section" or "|Section", so fix that pipe if i == 1: link_text = link_text.replace('|', '#') # Sometimes we use a space char. instead of a '_', so fix that before querying link_text = link_text.replace(' ', '_') if debug: pywikibot.stdout(' Found link {0}.'.format(link_text)) # If this link doesn't have a section link in it, then we don't care about it, as # MediaWiki takes care of checking basic intrawiki links if not '#' in link_text: if debug: pywikibot.stdout(' Link doesn\'t have a section anchor in it. Skipping.') continue # If this link has an interwiki prefix, it can be ignored; see check_interwiki_links.py # for the task of checking interwiki page+section links is_interwiki = False if found_iw_match == False: for prefix in interwiki_prefixes: if prefix + ":" in link_text: if debug: pywikibot.stdout(' Skipping link {} because it is an interwiki link.'.format(link_text)) is_interwiki = True break if is_interwiki: continue # If there is a '{' in the link, then probably it's a link built on transcluded text # like "Quotes/Diary#{{C3}}", which we cannot expand and work with, so skip it if '{' in link_text: ch_link_pattern = re.compile(r"{{C[0-9]*}}") ch_link = ch_link_pattern.search(link_text) if debug: pywikibot.stdout(' Found transclusion in link: "{}".'.format(ch_link.group(0))) if ch_link: ch_link_match = ch_link.group(0) ch_num_pattern = re.compile("[0-9]+") ch_num = ch_num_pattern.search(ch_link_match) if ch_num: ch_num_match = int(ch_num.group(0)) if ch_num_match >= 0 and ch_num_match <= 14: ch_name = chapter_names[ch_num_match] replace_pattern = re.compile(r"{{C" + ch_num.group(0) + r"}}") link_text = replace_pattern.sub(ch_name, link_text) if debug: pywikibot.stdout(' After performing transclusion, link is now "{}".'.format(link_text)) else: if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ADVICE: Link {0} transcludes a chapter name using an out-of-range number, {1}.'.format(link_text, ch_num_match)) advice_issued += 1 continue else: if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ADVICE: Link {} seems to be transcluding a chapter name, but this script couldn\'t read it.'.format(link_text)) advice_issued += 1 continue else: if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ADVICE: Link {0} seems to use transclusion. This script can understand chapter name transclusions such as "{1}" but it doesn\'t recognize this one so it can\'t be verified. You should check the link manually.'.format(link_text, "{{C7}}")) advice_issued += 1 continue # If this is a relative "/" link, use the current page as the basis for the URL. Note # that only a leading slash is looked for, so if there's multiple steps down ("/x/y"), # we're out of luck. if link_text.startswith('/'): link_text = page_name + link_text if debug: pywikibot.stdout(' Changed link_text to {} on account of "/".'.format(link_text)) # If this is a relative "../" link, find the parent page, set ourselves to that page, # then remove the relative portion of the link. Note that this is only performed once, # so if there's multiple steps back ("../../"), we're out of luck. if link_text.startswith('../'): last_slash = page_name.rfind('/') page_name2 = page_name[0:last_slash] if debug: pywikibot.stdout(' Changed page_name to {} on account of "../".'.format(page_name2)) link_text = link_text[3:len(link_text)] if debug: pywikibot.stdout(' Changed link_text to {} on account of "../".'.format(link_text)) # If this is now going to be a bare section link for the parent page, don't add a # slash, otherwise do because we are drilling down to another subpage if link_text.startswith('#'): link_text = page_name2 + link_text else: link_text = page_name2 + '/' + link_text # If this is a bare section link, build URL based on this page if link_text.startswith('#'): iw_url = onigalore_url + page_name2 iw_found += 1 if debug: pywikibot.stdout(' Found link to this very page, {}.'.format(link_text)) found_iw_match = True link_text = page_name2 + link_text # If there's no ":" in the link (before the section link, where a colon would just be # part of the text) then it's a Main namespace article; proceed with building URL if found_iw_match == False: if not re.search(":.*#", link_text): iw_url = onigalore_url + link_text iw_found += 1 if debug: pywikibot.stdout(' Link is to a Main namespace page.') found_iw_match = True # If there is a ":", match the prefix against the intrawiki prefixes on OniGalore # before building URL if found_iw_match == False: for prefix in intrawiki_prefixes: if prefix + ":" in link_text: iw_url = onigalore_url + link_text if debug: pywikibot.stdout(' Identified namespace {}.'.format(prefix)) iw_found += 1 found_iw_match = True break # If we still haven't turned this match into a URL, something's gone wrong if (found_iw_match == False) or (iw_url == ""): if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ERROR: Couldn\'t figure out link {}.'.format(link_text)) continue # Test the URL iw_url = iw_url.replace(' ', '_') if debug: pywikibot.stdout(' Reading page at {}...'.format(iw_url)) response = fetch(iw_url) # Redirects are followed automatically by fetch() and treated as "200"s; the way we can # tell that a redirect occurred is by checking fetch's history if response.history != []: if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ADVICE: Got redirection code ({0}) on URL "{1}". You should check the link manually.'.format(response.history[0], iw_url)) advice_issued += 1 elif response.status_code != 200: if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ERROR: Got response code {0} on URL {1}. The target page may not exist.'.format(response.status_code, iw_url)) errors_issued += 1 else: # Isolate section link pre_section, section_name = link_text.split('#', 1) if debug: pywikibot.stdout(' Searching for section link {} on page.'.format(section_name)) # Convert slash character to the dot-notation hex encoding that MediaWiki uses section_name = section_name.replace('/', '.2F') # Read linked page to see if it really has this anchor link soup = BeautifulSoup(response.text, 'html.parser') found_section = False for span_tag in soup.findAll('span'): span_name = span_tag.get('id', None) if span_name == section_name: if debug: pywikibot.stdout(' Found section!') found_section = True break if found_section == False: if not name_printed and not debug: pywikibot.stdout('From page "{}":'.format(page_name)) name_printed = 1 pywikibot.stdout(' ERROR: Could not find section {0} on page {1}!'.format(section_name, pre_section)) errors_issued += 1 def main(*args): global debug global pages_checked global iw_found global advice_issued global errors_issued search_cat = '' search_page = '' local_args = pywikibot.handle_args(args) genFactory = pagegenerators.GeneratorFactory() for arg in local_args: if arg.startswith('-cat:'): search_cat = arg[5:] elif arg.startswith('-page:'): search_page = arg[6:] elif arg == '-dbg': debug = 1 else: pywikibot.stdout('Unknown argument "{}".'.format(arg)) return site = pywikibot.Site() # This line of code enumerates the methods in the 'page' class #pywikibot.stdout(format(dir(page))) if search_cat != '': cat_obj = pywikibot.Category(site, search_cat) generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True) for page in pagegenerators.PreloadingGenerator(generator, 100): if debug: pywikibot.stdout('Checking page {0}'.format(page.title())) scan_for_intrawiki_links(page.text, page.title()) elif search_page != '': page = pywikibot.Page(site, search_page) if debug: pywikibot.stdout('Checking page {0}'.format(page.title())) scan_for_intrawiki_links(page.text, page.title()) page_str = "pages" if pages_checked == 1: page_str = "page" link_str = "links" if iw_found == 1: link_str = "link" pywikibot.stdout('Checked {0} {1} and found {2} intrawiki {3}.'.format(pages_checked, page_str, iw_found, link_str)) pywikibot.stdout('While attempting to follow section links...') if advice_issued == 0: pywikibot.stdout(' No advice on potential problems was issued.') elif advice_issued == 1: pywikibot.stdout(' 1 piece of advice on a potential problem was issued.') else: pywikibot.stdout(' {} pieces of advice on potential problems were issued.'.format(advice_issued)) error_str = "errors were" if errors_issued == 1: error_str = "error was" pywikibot.stdout(' {0} {1} encountered.'.format(errors_issued, error_str)) if __name__ == '__main__': main()