Index: /ValBot/Python/check_interwiki_links.py
===================================================================
--- /ValBot/Python/check_interwiki_links.py	(revision 1195)
+++ /ValBot/Python/check_interwiki_links.py	(revision 1196)
@@ -1,5 +1,5 @@
 # Check Interwiki Links
 # by iritscen@yahoo.com
-# Looks at each link on a page (or in all the pages in a category) which uses a registered
+# Looks at each link on a page (or all the pages in a category) which uses a registered
 # interwiki prefix and loads the linked page, verifying that it exists and that any section
 # link, if present, is valid as well. The output will use the word "ERROR" when it cannot
@@ -8,19 +8,25 @@
 # |---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---|
 
-import os
-
-from urllib.parse import urljoin
-
+import bs4
 import pywikibot
-import bs4
 import re
-import requests # for listing members with dir()
-
+import requests # for listing members with dir() when debugging
+
+from bs4 import BeautifulSoup
+from pywikibot import pagegenerators
 from pywikibot.bot import QuitKeyboardInterrupt
-from pywikibot import pagegenerators
-from pywikibot.tools.formatter import color_format
 from pywikibot.comms.http import fetch
 from pywikibot.specialbots import UploadRobot
-from bs4 import BeautifulSoup
+from pywikibot.tools.formatter import color_format
+from urllib.parse import urljoin
+
+class IWLink:
+   def __init__(self, iw_prefix, prefix_url, full_url, page_name, page_slug, curl_response):
+      self.iw_prefix = iw_prefix # e.g. "wp"
+      self.prefix_url = prefix_url # e.g. "https://en.wikipedia.org/wiki/"
+      self.full_url = full_url # e.g. "https://en.wikipedia.org/wiki/Easter_egg"
+      self.page_name = page_name # "Easter egg"
+      self.page_slug = page_slug # "Easter_egg"
+      self.curl_response = curl_response # a class defined in the Requests library
 
 # Parallel arrays based on https://wiki.oni2.net/Special:Interwiki
@@ -34,4 +40,5 @@
 iw_found = 0
 errors_issued = 0
+unintended_redirects_found = 0
 name_printed = 0
 
@@ -47,68 +54,49 @@
 
 # Search a page for the section specified in the link
-def find_section(page_text, page_name, page_slug, prefix, print_result):
+def find_section(the_link, print_result):
    global errors_issued
 
    # Isolate section link
-   target_page_name, anchor_name = page_slug.split('#')
+   target_page_name, anchor_name = the_link.page_slug.split('#')
    target_page_name_human = target_page_name.replace('_', ' ')
    
    # Convert dot-notation hex entities to proper characters
-   anchor_name = anchor_name.replace('.22', '"')
-   anchor_name = anchor_name.replace('.27', '\'')
-   anchor_name = anchor_name.replace('.28', '(')
-   anchor_name = anchor_name.replace('.29', ')')
+   replacements = [(r'\.22', '"'), (r'\.27', "'"), (r'\.28', '('), (r'\.29', ')')]
+   for pattern, replacement in replacements:
+      anchor_name = re.sub(pattern, replacement, anchor_name)
    
    # Read linked page to see if it really has this anchor link
-   soup = BeautifulSoup(page_text, 'html.parser')
+   soup = BeautifulSoup(the_link.curl_response.text, 'html.parser')
+   tags_to_search = ['span', 'div', 'h2', 'h3', 'h4']
    found_section = False
-   for the_tag in soup.findAll('span'): # search for span with ID matching the section name
-      tag_name = the_tag.get('id', None)
-      if tag_name == anchor_name:
-         found_section = True
-         break
+   for tag_name in tags_to_search:
+       for the_tag in soup.find_all(tag_name):
+           if the_tag.get('id') == anchor_name:
+               found_section = True
+               break
+       if found_section:
+           break
+   
+   # Tell user what we found
    if found_section == False:
-      for the_tag in soup.findAll('div'): # search for div with ID matching the section name
-         tag_name = the_tag.get('id', None)
-         if tag_name == anchor_name:
-            found_section = True
-            break
-   if found_section == False:
-      for the_tag in soup.findAll('h2'): # search for h2 with ID matching the section name
-         tag_name = the_tag.get('id', None)
-         if tag_name == anchor_name:
-            found_section = True
-            break
-   if found_section == False:
-      for the_tag in soup.findAll('h3'): # search for h3 with ID matching the section name
-         tag_name = the_tag.get('id', None)
-         if tag_name == anchor_name:
-            found_section = True
-            break
-   if found_section == False:
-      for the_tag in soup.findAll('h4'): # search for h4 with ID matching the section name
-         tag_name = the_tag.get('id', None)
-         if tag_name == anchor_name:
-            found_section = True
-            break
-   if found_section == False:
-      possibly_print(page_name)
-      pywikibot.stdout('   ERROR: Could not find section "{0}" on {1} page "{2}".'.format(anchor_name, prefix, target_page_name_human))
+      possibly_print(the_link.page_name)
+      pywikibot.stdout('   ERROR: Could not find section "{0}" on {1} page "{2}".'.format(anchor_name, the_link.iw_prefix, target_page_name_human))
+      # TODO: Check that page name has been corrected to redirected page if there was a redirect
       errors_issued = errors_issued + 1
    elif print_result == True:
-      pywikibot.stdout('   The section "{0}" was found on {1} page "{2}".'.format(anchor_name, prefix, target_page_name_human))
+      pywikibot.stdout('   The section "{0}" was found on {1} page "{2}".'.format(anchor_name, the_link.iw_prefix, target_page_name_human))
 
 # For a link that redirected us to another page, extract the name of the target page from
 # the target page's source
-def find_canonical_link(page_text, page_name, page_slug, prefix, prefix_url):
+def find_canonical_link(the_link):
    # Extract link from this markup which contains name of redirected-to page:
    # <link rel="canonical" href="https://en.wikipedia.org/wiki/Page_name"/>
-   canonical_name = page_text.split('<link rel="canonical" href="')[-1]
-   prefix_length = len(prefix_url)
+   canonical_name = the_link.curl_response.text.split('<link rel="canonical" href="')[-1]
+   prefix_length = len(the_link.prefix_url)
    canonical_name = canonical_name[prefix_length:]
    tag_end = canonical_name.find('">')
    
    if tag_end == -1:
-      pywikibot.stdout('   ERROR: The {0} link "{1}" is a redirect page, but this script could not isolate the target page name.'.format(prefix, page_slug))
+      pywikibot.stdout('   ERROR: The {0} link "{1}" is a redirect page, but this script could not isolate the target page name.'.format(the_link.iw_prefix, the_link.page_slug))
       errors_issued = errors_issued + 1
    else:
@@ -117,43 +105,46 @@
          # Certain things can cause the trim to fail; report error and avoid slamming the
          # output with massive page source from a failed trim
-         pywikibot.stdout('   ERROR: The {0} link "{1}" is a redirect to "{2}…" (string overflow).'.format(prefix, page_slug, canonical_name[:100]))
+         pywikibot.stdout('   ERROR: The {0} link "{1}" is a redirect to "{2}…" (string overflow).'.format(the_link.iw_prefix, the_link.page_slug, canonical_name[:100]))
          errors_issued = errors_issued + 1
       else:
          canonical_name = canonical_name.replace('_', ' ')
-         if '#' in page_slug:
-            _, anchor_name = page_slug.split('#')
-            pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}#{3}", which is a valid page. Checking section link….'.format(prefix, page_slug, canonical_name, anchor_name))
-            find_section(page_text, page_name, page_slug, prefix, True)
+         if '#' in the_link.page_slug:
+            _, anchor_name = the_link.page_slug.split('#')
+            pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}#{3}", which is a valid page. Checking for section on that page….'.format(the_link.iw_prefix, the_link.page_slug, canonical_name, anchor_name))
+            the_link.page_slug = the_link.page_slug.replace(the_link.page_name, canonical_name) # update page slug so that find_section() uses the right page name in its messages
+            find_section(the_link, True)
          else:
-            pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}", which is a valid page.'.format(prefix, page_slug, canonical_name))
+            pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}", which is a valid page.'.format(the_link.iw_prefix, the_link.page_slug, canonical_name))
 
 # Test an interwiki link and look for a section link if applicable
-def test_interwiki_link(prefix, prefix_url, iw_url, page_name, page_slug):
+def test_interwiki_link(the_link):
    global errors_issued
-   
-   response = fetch(iw_url)
+   global unintended_redirects_found
+   
+   the_link.curl_response = fetch(the_link.full_url)
 
    # One way we tell that a redirect occurred is by checking fetch's history, as it
    # automatically follows redirects. This will catch formal redirects which come from pages
    # such as Special:PermanentLink.
-   if response.history != []:
-      possibly_print(page_name)
-         
-      if page_slug.startswith('WP:') and page_slug == page_slug.upper():
-         pywikibot.stdout('   Got redirection code "{0}" for {1} link "{2}". This appears to be a deliberate use of a Wikipedia shortcut. Checking the target page….'.format(response.history[0], prefix, page_slug))
-         find_canonical_link(response.text, page_name, page_slug, prefix, prefix_url)
+   if the_link.curl_response.history != []:
+      possibly_print(the_link.page_name)
+      
+      # If linked page is in all caps, e.g. WP:BEANS, it's likely a deliberate use of a redirect
+      if the_link.page_slug.startswith('WP:') and the_link.page_slug == the_link.page_slug.upper():
+         pywikibot.stdout('   Got redirection code "{0}" for {1} link "{2}". This appears to be a deliberate use of a Wikipedia shortcut. Checking the target page….'.format(the_link.curl_response.history[0], the_link.iw_prefix, the_link.page_slug))
+         find_canonical_link(the_link)
       else:
          permalink1 = 'Special:PermanentLink/'.lower()
          permalink2 = 'Special:Permalink/'.lower()
-         page_slug_lower = page_slug.lower()
+         page_slug_lower = the_link.page_slug.lower()
          if page_slug_lower.startswith(permalink1) or page_slug_lower.startswith(permalink2):
-            pywikibot.stdout('   Got redirection code "{0}" for {1} permanent revision link "{2}". Checking the target page….'.format(response.history[0], prefix, page_slug))
-            find_canonical_link(response.text, page_name, page_slug, prefix, prefix_url)
+            pywikibot.stdout('   Got redirection code "{0}" for {1} permanent revision link "{2}". Checking the target page….'.format(the_link.curl_response.history[0], the_link.iw_prefix, the_link.page_slug))
+            find_canonical_link(the_link)
          else:
-            pywikibot.stdout('   ERROR: Unrecognized type of redirection (code "{0}") for {1} link "{2}". You should check the link manually.'.format(response.history[0], prefix, page_slug))
+            pywikibot.stdout('   ERROR: Unrecognized type of redirection (code "{0}") for {1} link "{2}". You should check the link manually.'.format(the_link.curl_response.history[0], the_link.iw_prefix, the_link.page_slug))
             errors_issued = errors_issued + 1
-   elif response.status_code != 200:
-      possibly_print(page_name)
-      pywikibot.stdout('   ERROR: Got response code {0} for {1} link "{2}". The page may not exist.'.format(response.status_code, prefix, page_slug))
+   elif the_link.curl_response.status_code != 200:
+      possibly_print(the_link.page_name)
+      pywikibot.stdout('   ERROR: Got response code {0} for {1} link "{2}". The page may not exist.'.format(the_link.curl_response.status_code, the_link.iw_prefix, the_link.page_slug))
       errors_issued = errors_issued + 1
    # However the usual way that a redirect occurs is that MediaWiki redirects us sneakily
@@ -161,10 +152,11 @@
    # when a redirect page is accessed. We must detect these soft redirects by looking at the
    # page source to find the redirect note inserted at the top of the page for the reader.
-   elif 'Redirected from <a' in response.text:
-      possibly_print(page_name)
-      pywikibot.stdout('   Got silently redirected by {0} link "{1}". Checking the target page….'.format(prefix, page_slug))
-      find_canonical_link(response.text, page_name, page_slug, prefix, prefix_url)
-   elif '#' in page_slug:
-      find_section(response.text, page_name, page_slug, prefix, False)
+   elif 'Redirected from <a' in the_link.curl_response.text:
+      unintended_redirects_found = unintended_redirects_found + 1
+      possibly_print(the_link.page_name)
+      pywikibot.stdout('   WARNING: Got silently redirected by {0} link "{1}". Checking the target page….'.format(the_link.iw_prefix, the_link.page_slug))
+      find_canonical_link(the_link)
+   elif '#' in the_link.page_slug:
+      find_section(the_link, False)
 
 # Searches the given page text for interwiki links
@@ -182,29 +174,31 @@
       iw_link = r"\[\[" + prefix + r":[^|\]]*(\||\])"
       for match in re.finditer(iw_link, page_text):
+         the_link = IWLink(prefix, interwiki_urls[cur_prefix], "", page_name, "", "")
+      
          # Extract just the page title from this regex match
-         s = match.start() + 2 + len(prefix) + 1
+         s = match.start() + 2 + len(the_link.iw_prefix) + 1
          e = match.end() - 1
 
          # Commonly we use spaces instead of underscores, so fix that before querying
-         page_slug = page_text[s:e].replace(' ', '_')
+         the_link.page_slug = page_text[s:e].replace(' ', '_')
 
          # But use spaces for title when printing it
-         page_title_human = page_slug.replace('_', ' ')
-         if debug: pywikibot.stdout('   Validating {0} link "{1}"'.format(prefix, page_title_human))
+         page_title_human = the_link.page_slug.replace('_', ' ')
+         if debug: pywikibot.stdout('   Validating {0} link "{1}"'.format(the_link.iw_prefix, page_title_human))
          iw_found = iw_found + 1
 
          # Construct full URL for the particular wiki
-         iw_url = interwiki_urls[cur_prefix] + page_slug
+         the_link.full_url = the_link.prefix_url + the_link.page_slug
 
          # Adjust URL if this is a foreign-language WP link
-         if re.match("^[a-zA-Z]{2}:", page_slug):
-            lang_code = page_slug[0:2] + "."
+         if re.match("^[a-zA-Z]{2}:", the_link.page_slug):
+            lang_code = the_link.page_slug[0:2] + "."
             # "wp:" is the Wikipedia: namespace, not a language
             if lang_code != "wp." and lang_code != "WP.":
-               iw_url = iw_url.replace('en.', lang_code)
-               iw_url = iw_url.replace(page_slug[0:3], '')
+               the_link.full_url = the_link.full_url.replace('en.', lang_code)
+               the_link.full_url = the_link.full_url.replace(the_link.page_slug[0:3], '')
 
          # Test the URL
-         test_interwiki_link(prefix, interwiki_urls[cur_prefix], iw_url, page_name, page_slug)
+         test_interwiki_link(the_link)
       cur_prefix = cur_prefix + 1
 
@@ -214,4 +208,5 @@
    global iw_found
    global errors_issued
+   global unintended_redirects_found
 
    page_str = "pages"
@@ -230,4 +225,10 @@
 
    pywikibot.stdout('{0} {1} encountered in validating these links.'.format(errors_issued, error_str))
+
+   warning_str = "likely-unintended redirects were"
+   if unintended_redirects_found == 1:
+      warning_str = "likely-unintended redirect was"
+
+   pywikibot.stdout('{0} {1} encountered in validating these links.'.format(unintended_redirects_found, warning_str))
 
 # Main function