Index: /ValBot/check_interwiki_links.py
===================================================================
--- /ValBot/check_interwiki_links.py	(revision 1151)
+++ /ValBot/check_interwiki_links.py	(revision 1151)
@@ -0,0 +1,105 @@
+import os
+
+from urllib.parse import urljoin
+
+import pywikibot
+import re
+
+from pywikibot.bot import QuitKeyboardInterrupt
+from pywikibot import pagegenerators
+from pywikibot.tools.formatter import color_format
+from pywikibot.comms.http import fetch
+from pywikibot.specialbots import UploadRobot
+from bs4 import BeautifulSoup
+
+# Parallel arrays based on https://wiki.oni2.net/Special:Interwiki
+interwiki_prefixes = ('acronym', 'cache', 'commons', 'dictionary', 'google', 'metawikimedia', 'mw', 'wikibooks', 'wikidata', 'wikimedia', 'wikinews', 'wikipedia', 'wikiquote', 'wikisource', 'wikispecies', 'wikiversity', 'wikivoyage', 'wikt', 'wiktionary', 'wp')
+
+interwiki_urls = ('http://www.acronymfinder.com/~/search/af.aspx?string=exact&Acronym=', 'http://www.google.com/search?q=cache:', 'https://commons.wikimedia.org/wiki/', 'http://www.dict.org/bin/Dict?Database=*&Form=Dict1&Strategy=*&Query=', 'http://www.google.com/search?q=', 'https://meta.wikimedia.org/wiki/', 'https://www.mediawiki.org/wiki/', 'https://en.wikibooks.org/wiki/', 'https://www.wikidata.org/wiki/', 'https://foundation.wikimedia.org/wiki/', 'https://en.wikinews.org/wiki/', 'https://en.wikipedia.org/wiki/', 'https://en.wikiquote.org/wiki/', 'https://wikisource.org/wiki/', 'https://species.wikimedia.org/wiki/', 'https://en.wikiversity.org/wiki/', 'https://en.wikivoyage.org/wiki/', 'https://en.wiktionary.org/wiki/', 'https://en.wiktionary.org/wiki/', 'https://en.wikipedia.org/wiki/')
+
+pages_checked = 0
+iw_found = 0
+problems_found = 0
+
+# Searches the given page text for interwiki links
+def scan_for_iw_links(page_text):
+    global pages_checked
+    global iw_found
+    global problems_found
+    pages_checked = pages_checked + 1
+    cur = 0
+
+    for prefix in interwiki_prefixes:
+        # Isolate strings that start with "[[prefix:" and end with "|" or "]"
+        iw_link = "\[\[" + prefix + ":[^|\]]*(\||\])"
+        for match in re.finditer(iw_link, page_text):
+            # Extract just the page title from this regex match
+            s = match.start() + 2 + len(prefix) + 1
+            e = match.end() - 1
+
+            # Sometimes we used a space char. instead of a '_', so fix that before querying
+            page_title = page_text[s:e].replace(' ', '_')
+
+            # Construct full URL for the particular wiki
+            iw_url = interwiki_urls[cur] + page_title
+            pywikibot.output('Found {0} link {1}'.format(prefix, page_title))
+            iw_found = iw_found + 1
+
+            # Adjust URL if this is a foreign-language WP link
+            if re.match("^[a-zA-Z]{2}:", page_title):
+                lang_code = page_title[0:2] + "."
+                # "wp:" is the Wikipedia: namespace, not a language
+                if lang_code != "wp." and lang_code != "WP.":
+                    iw_url = iw_url.replace('en.', lang_code)
+                    iw_url = iw_url.replace(page_title[0:3], '')
+
+            # Test the URL
+            #pywikibot.output('Testing URL {}'.format(iw_url))
+            response = fetch(iw_url)
+
+            # Redirects are followed automatically by fetch() and treated as "200"s, so the
+            # way we tell that a redirect occurred is by checking the history
+            if response.history != []:
+                pywikibot.output('WARNING: Initially got {}.'.format(response.history))
+                problems_found = problems_found + 1
+            elif response.status_code != 200:
+                #pywikibot.output('WARNING: Got response code {}.'.format(response.status_code)) # commented out because fetch() already prints such a msg
+                problems_found = problems_found + 1
+        cur = cur + 1
+
+def main(*args):
+    cat_name = ''
+    page_name = ''
+
+    local_args = pywikibot.handle_args(args)
+    genFactory = pagegenerators.GeneratorFactory()
+
+    for arg in local_args:
+        if arg.startswith('-cat:'):
+            cat_name = arg[5:]
+        elif arg.startswith('-page:'):
+            page_name = arg[6:]
+
+    site = pywikibot.Site()
+
+    # This line of code enumerates the methods in the 'page' class
+    #pywikibot.stdout(format(dir(page)))
+
+    if cat_name != '':
+        cat_obj = pywikibot.Category(site, cat_name)
+        generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True)
+        for page in pagegenerators.PreloadingGenerator(generator, 100):
+            pywikibot.stdout('Checking page {0}'.format(page.title()))
+            scan_for_iw_links(page.text)
+    elif page_name != '':
+        page = pywikibot.Page(site, page_name)
+        pywikibot.stdout('Checking page {0}'.format(page.title()))
+        scan_for_iw_links(page.text)
+
+    global pages_checked
+    global iw_found
+    global problems_found
+    pywikibot.stdout('Checked {0} page(s) and found {1} interwiki link(s) with {2} problem(s).'.format(pages_checked, iw_found, problems_found))
+
+if __name__ == '__main__':
+    main()
Index: /ValBot/drive_https_upgrade.sh
===================================================================
--- /ValBot/drive_https_upgrade.sh	(revision 1151)
+++ /ValBot/drive_https_upgrade.sh	(revision 1151)
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+IFS="
+"
+
+CORE="/path/to/Pywikibot/core"
+SUMMARY="changing link from http->https"
+RATE=6
+FIX_START=551
+FIX_END=650
+
+cd "$CORE"
+if [ ! -f "pwb.py" ]; then
+   echo "drive_https_upgrade.sh: Can't launch Pywikibot!"
+   exit
+fi
+
+echo "drive_https_upgrade.sh: Starting at fix $FIX_START..."
+
+FIX_CUR=0
+LAST_RUN=0
+for THE_LINE in `cat "/path/to/ValExtLinks report.txt"`; do
+   if [[ $THE_LINE =~ .*http-\>https.* ]]; then
+      let FIX_CUR+=1
+      if [ $FIX_CUR -lt $FIX_START ]; then
+         continue
+      fi
+      if [ $FIX_CUR -gt $FIX_END ]; then
+         echo "drive_https_upgrade.sh: Stopped after fix $FIX_END."
+         exit
+      fi
+
+      # Wait for rate limit to expire if we have run the Python script before in this session
+      if [ $LAST_RUN -gt 0 ]; then
+         CUR_TIME=$(date +%s)
+         WAIT_REMAINDER=$(($RATE - $CUR_TIME + $LAST_RUN))
+         if [ $WAIT_REMAINDER -gt 0 ]; then
+            echo "drive_https_upgrade.sh: Waiting $WAIT_REMAINDER second(s)."
+            sleep $WAIT_REMAINDER
+         fi
+      fi
+      ON_PAGE=${THE_LINE#*page \'}
+      ON_PAGE=${ON_PAGE%%\'*}
+      FROM_LINK=${THE_LINE#*URL \'}
+      FROM_LINK=${FROM_LINK%%\'*}
+      TO_LINK=${THE_LINE%\'*}
+      TO_LINK=${TO_LINK##*\'}
+      LAST_RUN=$(date +%s)
+      python pwb.py replace -page:"$ON_PAGE" "$FROM_LINK" "$TO_LINK" -summary:"$SUMMARY" -always
+   fi
+done
Index: /ValBot/drive_slash_adding.sh
===================================================================
--- /ValBot/drive_slash_adding.sh	(revision 1151)
+++ /ValBot/drive_slash_adding.sh	(revision 1151)
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+IFS="
+"
+
+CORE="/path/to/Pywikibot/core"
+SUMMARY="added ending slash to URL and/or upgrading http to https to satisfy redirect"
+RATE=6
+FIX_START=0
+FIX_END=0
+
+cd "$CORE"
+if [ ! -f "pwb.py" ]; then
+   echo "drive_slash_adding.sh: Can't launch Pywikibot!"
+   exit
+fi
+
+echo "drive_slash_adding.sh: Starting at fix $FIX_START..."
+
+FIX_CUR=0
+LAST_RUN=0
+for THE_LINE in `cat "/path/to/ValExtLinks report.txt"`; do
+   #echo "drive_slash_adding.sh: Considering '$THE_LINE'..."
+   if [[ "$THE_LINE" =~ .*trailing.* ]] && [[ ! "$THE_LINE" =~ .*w/index.php.* ]]; then
+      #echo "drive_slash_adding.sh: This URL needs to be fixed."
+      let FIX_CUR+=1
+
+      if [ $FIX_CUR -lt $FIX_START ]; then
+         continue
+      fi
+
+      if [ $FIX_END -gt 0 ] && [ $FIX_CUR -gt $FIX_END ]; then
+         echo "drive_slash_adding.sh: Stopped after fix $FIX_END."
+         exit
+      fi      
+
+      # Wait for rate limit to expire if we have run the Python script before in this session
+      if [ $LAST_RUN -gt 0 ]; then
+         CUR_TIME=$(date +%s)
+         WAIT_REMAINDER=$(($RATE - $CUR_TIME + $LAST_RUN))
+         if [ $WAIT_REMAINDER -gt 0 ]; then
+            echo "drive_slash_adding.sh: Waiting $WAIT_REMAINDER second(s)."
+            sleep $WAIT_REMAINDER
+         fi
+      fi
+      ON_PAGE=${THE_LINE#*page \'}
+      ON_PAGE=${ON_PAGE%%\'*}
+      FROM_LINK=${THE_LINE#*URL \'}
+      FROM_LINK=${FROM_LINK%%\'*}
+      TO_LINK=${THE_LINE%\'*}
+      TO_LINK=${TO_LINK##*\'}
+
+      #if [[ "$THE_LINE" =~ ${FROM_LINK}[^a-zA-Z/] ]]; then
+      #   echo "URL is not isolated, skipping."
+      #   continue
+      #fi
+
+      LAST_RUN=$(date +%s)
+      echo "pwb.by replace '-page:\"$ON_PAGE\" \"$FROM_LINK\" \"$TO_LINK\""
+      python pwb.py replace -page:"$ON_PAGE" "$FROM_LINK" "$TO_LINK" -summary:"$SUMMARY"
+   fi
+done
Index: /ValBot/find_external_images.py
===================================================================
--- /ValBot/find_external_images.py	(revision 1151)
+++ /ValBot/find_external_images.py	(revision 1151)
@@ -0,0 +1,89 @@
+import os
+
+from urllib.parse import urljoin
+
+import pywikibot
+
+from pywikibot.bot import QuitKeyboardInterrupt
+from pywikibot import pagegenerators
+from pywikibot.comms.http import fetch
+from pywikibot.specialbots import UploadRobot
+from bs4 import BeautifulSoup
+
+first_run = False
+pages_checked = 0
+oni2_images = 0
+file_formats = ('.jpg', '.jpeg', '.png', '.gif', '.svg', '.ogg')
+
+# Scrapes the HTML at the given URL for image tags
+def get_image_links(url, shown):
+    links = []
+    global oni2_images
+    global pages_checked
+
+    response = fetch(url)
+    if response.status_code != 200:
+        pywikibot.output('Skipping url: {}'.format(url))
+        return links
+
+    soup = BeautifulSoup(response.text, 'html.parser')
+    pages_checked = pages_checked + 1
+    if not shown:
+        tagname = 'a'
+    elif shown == 'just':
+        tagname = 'img'
+    else:
+        tagname = ['a', 'img']
+    #pywikibot.output('Looking at tags.')
+    for tag in soup.findAll(tagname):
+        link = tag.get('src', tag.get('href', None))
+        if not link:
+            #pywikibot.output('It is not a link.')
+            continue
+        #pywikibot.output('Got link {0}.'.format(link))
+        _, ext = os.path.splitext(link)
+        if ext.lower() in file_formats:
+            pywikibot.output('Found image link {0}.'.format(ext))
+            if "oni2.net" in link:
+                pywikibot.stdout('Found an oni2.net image: {0}'.format(link))
+                oni2_images = oni2_images + 1
+    return links
+
+
+def main(*args):
+    cat = ''
+    url = ''
+    image_url = False
+    shown = False
+    desc = []
+
+    local_args = pywikibot.handle_args(args)
+    genFactory = pagegenerators.GeneratorFactory()
+
+    for arg in local_args:
+        if arg.startswith('-cat:'):
+            cat = arg[5:]
+        elif arg == '-shown':
+            shown = True
+        elif arg == '-justshown':
+            shown = 'just'
+        elif url == '':
+            url = arg
+        else:
+            desc += [arg]
+    desc = ' '.join(desc)
+
+    site = pywikibot.Site()
+    cat_obj = pywikibot.Category(site, cat)
+    generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True)
+    for page in pagegenerators.PreloadingGenerator(generator, 100):
+        pywikibot.stdout('Checking page {0}'.format(page.title()))
+        page_url = page.full_url().replace("%2F", "/")
+        get_image_links(page_url, shown)
+
+    global pages_checked
+    global oni2_images
+    pywikibot.stdout('Checked {0} page(s) and found {1} image(s) from oni2.net.'.format(pages_checked, oni2_images))
+
+if __name__ == '__main__':
+    main()