Changeset 1175 for Validate External Links/validate_external_links.sh

Timestamp:

Aug 23, 2022, 4:15:48 PM (2 years ago)

Author:

iritscen

Message:

ValExtLinks: Added audit feature which tells the user if there are items in the exception list which are no longer present on the wiki or no longer return the given error code.

File:

• Validate External Links/validate_external_links.sh (modified) (14 diffs)

Validate External Links/validate_external_links.sh

@@ -30 +30 @@
 ### GLOBALS ###
 # Settings -- these will be changed from their defaults by the arguments passed in to the script
-LINKS_URL=""           # use 'curl' to download file with links from this location (can be file://)
-EXCEPT_URL=""          # 'curl' will access this wiki page with a list of exceptions for NG results
+LINKS_URL=""           # download external link CSV from this location (can use "file://" protocol)
+EXCEPT_URL=""          # location of wiki page with a list of exceptions for NG results
 OUTPUT_DIR=""          # place reports and all other output in a folder inside this existing folder
 RECORD_OK_LINKS=0      # record response code to the log even when it's a value in OK_CODES
@@ -48 +48 @@
 
 # Fixed strings -- see the occurrences of these variables to learn their purpose
-AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/88.0.4324.146 Safari/537.36"
+AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36"
 ARCHIVE_API="http://archive.org/wayback/available"
 ARCHIVE_GENERIC="https://web.archive.org/web/*"
@@ -69 +69 @@
 # These arrays tell the script which suffixes at the ends of URLs represent files and which are pages.
 # This determines whether the script tries to take a screenshot of the URL or just gets its HTTP code.
-declare -a HTTP_FILES=(3ds 7z avi BINA blend bsl cpp csv dae dll dmg doc east exe fbx first63 flv gamedata gif jpg last32 log m4a mhm mov mp3 mp4 oni ONWC pdf png psd py rar tga TRMA txt vbs wav wmv xaf xcf xml zip)
+declare -a HTTP_FILES=(3ds 7z avi BINA blend bsl cpp csv dae dll dmg doc east exe fbx first63 flv gamedata gif jpg last32 log m4a mhm mov mp3 mp4 oni ONWC pdf png psd py rar tga TRMA txt vbs wav wmv xaf xcf xlsx xml zip)
 declare -a HTTP_TLDS_AND_PAGES=(abstract action ars asp aspx cfm cgi com css de do full htm html it js jsp net org pgi php php3 phtml pl ru shtml stm uk x)
 
@@ -563 +563 @@
    if [ $SKIP_EXPECT_IW -gt 0 ]; then valPrint ctrh "- $SKIP_EXPECT_IW/$IW_LINKS potential intrawiki $(pluralCheckNoun link $IW_LINKS)"; fi
 
+   # Perform exceptions audit
+   EXCEPTION_ISSUES=0
+   valPrint ctrh "Exceptions list audit:"
+   for ((i = 0; i < ${#EXCEPT_ARRAY[@]}; ++i)); do
+      EXCEPT_LINE="${EXCEPT_ARRAY[$i]}"
+      EXCEPT_LINE=$(echo "$EXCEPT_LINE" | sed 's/\&amp;/\&/g') # copied from exception-matching code
+
+      if [ ${EXCEPT_FOUND[$i]} -eq 0 ]; then
+         EXCEPT_URL="${EXCEPT_LINE#*,}"
+         EXCEPT_URL="${EXCEPT_URL%,*}"
+         EXCEPT_PAGE="${EXCEPT_LINE##*,}"
+         EXCEPT_PAGE="${EXCEPT_PAGE%% *}"
+         if [ "$EXCEPT_PAGE" == "*" ]; then
+            valPrint tr "- The link '$EXCEPT_URL' did not return an error or is not on any page."
+         else
+            valPrint tr "- The link '$EXCEPT_URL' did not return an error or is not on page '$EXCEPT_PAGE'."
+         fi
+         let EXCEPTION_ISSUES+=1
+      elif [ ${EXCEPT_USED[$i]} -eq 0 ]; then
+         EXCEPT_URL="${EXCEPT_LINE#*,}"
+         EXCEPT_URL="${EXCEPT_URL%,*}"
+         EXCEPT_CODE=${EXCEPT_LINE%%,*}
+         valPrint tr "- The link '$EXCEPT_URL' did not return error code $EXCEPT_CODE."
+         let EXCEPTION_ISSUES+=1
+      fi
+   done
+   if [ $EXCEPTION_ISSUES -eq 0 ]; then
+      valPrint ctrh "- No issues found."
+   else
+      valPrint c "$EXCEPTION_ISSUES exception list $(pluralCheckNoun issue $EXCEPTION_ISSUES) detected (see RTF or TXT report for details)."
+      valPrint h "$EXCEPTION_ISSUES exception list $(pluralCheckNoun issue $EXCEPTION_ISSUES) detected (see <a href=\"$LOG_NAME_RTF\" target=\"_blank\">RTF</a> or <a href=\"$LOG_NAME_TXT\" target=\"_blank\">TXT</a> report for details)."
+   fi
+
    # Print checked link totals
    if [ $LINK_PROBLEMS_NET -gt 0 ]; then valPrint ctrh "$LINK_PROBLEMS_NET link $(pluralCheckNoun issue $LINK_PROBLEMS_NET):"; fi
@@ -630 +663 @@
    # Transfer to array for easy searching later
    declare -a EXCEPT_ARRAY=($(echo "$EXCEPT_DATA"))
+
+   # Create parallel arrays for marking which exceptions get used later
+   declare -a EXCEPT_USED=()
+   declare -a EXCEPT_FOUND=()
+   for ((i = 0; i < ${#EXCEPT_ARRAY[@]}; ++i)); do
+      EXCEPT_USED+=(0)
+      EXCEPT_FOUND+=(0)
+   done
 fi
 
@@ -689 +730 @@
 valPrint r "\b1 Legend \b0"
 valPrint hn "<h3>Legend</h3>"
-valPrint t "(For guidance in fixing these links, see $WIKI_MAIN.)"
-valPrint r "(For guidance in fixing these links, see {\field{\*\fldinst{HYPERLINK \"$WIKI_MAIN\"}}{\fldrslt here}}.)"
-valPrint h "(For guidance in fixing these links, see <a href=\"$WIKI_MAIN\" target=\"_blank\">here</a>.)"
+valPrint t "(For guidance in fixing these links, see $WIKI_MAIN. The exceptions list is at $EXCEPT_URL.)"
+valPrint r "(For guidance in fixing these links, see {\field{\*\fldinst{HYPERLINK \"$WIKI_MAIN\"}}{\fldrslt here}}. The exceptions list is {\field{\*\fldinst{HYPERLINK \"$EXCEPT_URL\"}}{\fldrslt here}}.)"
+valPrint h "(For guidance in fixing these links, see <a href=\"$WIKI_MAIN\" target=\"_blank\">here</a>. The exceptions list is <a href=\"$EXCEPT_URL\" target=\"_blank\">here</a>.)"
 valPrint trh "OK = URL seems to be working"
 valPrint trh "NG = URL no longer seems to work"
@@ -829 +870 @@
    CLEAN_URL=${CLEAN_URL%%\#*}
 
-   # 'sed' cannot handle Unicode in my Bash shell, so skip non-ASCII URL and make user check it
+   # 'sed' cannot handle Unicode in my Bash shell, so skip non-ASCII URL and make reader check it
    if [[ $CLEAN_URL == *[![:ascii:]]* ]]; then
       valPrint trs "Skipping URL '$URL' (found on page '$LOCAL_PAGE_PATH') because I cannot handle non-ASCII characters."
@@ -902 +943 @@
    fi
 
-   # If this suffix escaped identification as either a file, page or TLD, inform the user
+   # If this suffix escaped identification as either a file, page or TLD, inform the reader
    STR_TYPE=""
    if [ $IS_FILE -eq -1 ]; then
@@ -1065 +1106 @@
    # Check problem links against exceptions list before proceeding
    FOUND_EXCEPT=0
-   if [ $STATUS != "OK" ] && [ ! -z $EXCEPT_URL ]; then
+   if [ $STATUS != "OK" ] && [ ! -z "$EXCEPT_URL" ]; then
       # The code we expect to find in the exceptions file is either the 'curl' result or "EI"/"IW"
       EXPECT_CODE="$CURL_RESULT"
@@ -1083 +1124 @@
          EXCEPT_LINE=$(echo "$EXCEPT_LINE" | sed 's/\&amp;/\&/g')
 
-         # Match URL
+         # Check for URL match
          EXCEPT_URL="${EXCEPT_LINE#*,}"
          EXCEPT_URL="${EXCEPT_URL%,*}"
@@ -1090 +1131 @@
          fi
 
-         # Match containing page's name
+         # Check for page name match
          EXCEPT_PAGE="${EXCEPT_LINE##*,}"
          EXCEPT_PAGE="${EXCEPT_PAGE%% *}"
-         if [ "$EXCEPT_PAGE" == "*" ] || [ "$EXCEPT_PAGE" == $LOCAL_PAGE_PATH ]; then
-            # Match result code
+         if [ "$EXCEPT_PAGE" == "*" ] || [ "$EXCEPT_PAGE" == "$LOCAL_PAGE_PATH" ]; then
+            let EXCEPT_FOUND[$i]+=1
+            valPrint trs "Found exception '$URL' on page '$LOCAL_PAGE_PATH'."
+
+            # Check for result code match
             EXCEPT_CODE=${EXCEPT_LINE%%,*}
             if [ "$EXCEPT_CODE" == "$EXPECT_CODE" ]; then
+               FOUND_EXCEPT=1
+               let EXCEPT_USED[$i]+=1
                valPrint trs "Skipping URL '$URL' (found on page '$LOCAL_PAGE_PATH') because its expected result, '$EXPECT_CODE', is in the exceptions list."
+
                if [ $STATUS == "EI" ]; then
                   let SKIP_EXPECT_EI+=1
@@ -1107 +1154 @@
                   let SKIP_EXPECT_NG+=1
                fi
-               FOUND_EXCEPT=1
+
                break
             fi
@@ -1181 +1228 @@
          ARCHIVE_QUERY=$(curl --silent --max-time $TIMEOUT "$ARCHIVE_API?url=$URL&$ARCHIVE_OK_CODES")
 
-         # Notify user if we hit the rate limit and just keep going
+         # Notify reader if we hit the rate limit and just keep going
          if [[ "$ARCHIVE_QUERY" == "*Too Many Requests*" ]]; then
             valPrint t "  IA has rate-limited us!"
             valPrint r "                IA has rate-limited us!"
             valPrint hs "<tr><td colspan=\"2\" align=\"right\">IA suggests</td><td>(hit the API rate limit!)</td></tr>"
-         # If a "closest" snapshot was received, inform user
+         # If a "closest" snapshot was received, inform reader
          elif [[ "$ARCHIVE_QUERY" == *\"closest\":* ]]; then
             # In case the URL has a shebang in it (like mega.nz links do), escape the '!' to break it
@@ -1199 +1246 @@
             SNAPSHOT_URL=$(echo $SNAPSHOT_URL | sed 's/:80//')
 
-            # Inform the user of the snapshot URL
+            # Inform the reader of the snapshot URL
             valPrint ts "  IA suggests $SNAPSHOT_URL"
             valPrint rs "               IA suggests {\field{\*\fldinst{HYPERLINK \"$SNAPSHOT_URL\"}}{\fldrslt $SNAPSHOT_URL}}"