Changeset 1184 for Validate External Links

Timestamp:

May 22, 2023, 12:22:55 AM (18 months ago)

Author:

iritscen

Message:

ValExtLinks: Added --only-200-ok argument.

Location:

Validate External Links

Files:

• Development/ValExtLinks to-do.rtf (modified) ( previous)
• validate_external_links.command (modified) (1 diff)
• validate_external_links.sh (modified) (8 diffs)

Validate External Links/validate_external_links.command

@@ -35 +35 @@
 
 # Normal run with no upload
-#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --show-https-upgrades --show-added-slashes
+#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --show-https-upgrades --show-added-slashes --show-yt-redirects
 
 # Normal run

Validate External Links/validate_external_links.sh

@@ -34 +34 @@
 OUTPUT_DIR=""           # place reports and all other output in a folder inside this existing folder
 RECORD_OK_LINKS=0      # record response code to the log even when it's a value in OK_CODES
+ONLY_200_OK=0          # only treat code 200 as "OK" and not any other code in OK_CODES
 SHOW_SLASH=0           # record issue when a slash is added to the end of a URL
 SHOW_HTTPS=0           # record issue when "http" is upgraded to "https"
@@ -68 +69 @@
 
 # These arrays tell the script which suffixes at the ends of URLs represent files and which are pages.
-# This determines whether the script tries to take a screenshot of the URL or just gets its HTTP code.
+# This determines whether the script tries to take a screenshot of the URL (when screenshots are
+# requested).
 declare -a HTTP_FILES=(3ds 7z avi BINA blend bsl cpp csv dae dll dmg doc east exe fbx first63 flv gamedata gif jpg last32 log m4a mhm mov mp3 mp4 oni ONWC pdf png psd py rar tga TRMA txt vbs wav wmv xaf xcf xlsx xml zip)
 declare -a HTTP_TLDS_AND_PAGES=(abstract action ars asp aspx cfm cgi com css de do full htm html it js jsp net org pgi php php3 phtml pl ru shtml stm uk x)
@@ -135 +137 @@
        validate_external_links.sh --help
        validate_external_links.sh --links URL --output DIR [--exceptions URL]
-          [--record-ok-links] [--show-added-slashes] [--show-https-upgrades]
-          [--show-yt-redirects] [--suggest-snapshots] [--check-archive-links]
-          [--take-screenshots FILE] [--timeout NUM] [--start-url NUM]
-          [--end-url NUM] [--upload FILE]
+          [--record-ok-links] [--only-200-ok] [--show-added-slashes]
+          [--show-https-upgrades] [--show-yt-redirects] [--suggest-snapshots]
+          [--check-archive-links] [--take-screenshots FILE] [--timeout NUM]
+          [--start-url NUM] [--end-url NUM] [--upload FILE]
 
 DESCRIPTION
@@ -173 +175 @@
        --record-ok-links       Log a link in the report even if its response
                                code is "OK".
+       --only-200-ok           Only treat response code 200 as "OK". Normally
+                               several additional codes are treated as "OK" (see
+                               the array OK_CODES in script) because they are
+                               typically not an indicator of a bad link.
        --show-added-slashes    Report on redirects that simply add a '/' to the
                                end of the URL.
@@ -226 +232 @@
       --output )               OUTPUT_DIR="$2";                    shift 2;;
       --record-ok-links )      RECORD_OK_LINKS=1;                  shift;;
+      --only-200-ok )          ONLY_200_OK=1;                      shift;;
       --show-added-slashes )   SHOW_SLASH=1;                       shift;;
       --show-https-upgrades )  SHOW_HTTPS=1;                       shift;;
@@ -704 +711 @@
 valPrint ctrhn "Show OK links: "
 if [ $RECORD_OK_LINKS -eq 1 ]; then valPrint ctrh "Yes"; else valPrint ctrh "No"; fi
+
+valPrint ctrhn "Treat these response codes as OK: "
+if [ $ONLY_200_OK -eq 1 ]; then valPrint ctrh "200"; else valPrint ctrh "${OK_CODES[*]}"; fi
 
 valPrint ctrhn "Take screenshots: "
@@ -1007 +1017 @@
    if [ $STATUS == "??" ]; then
       for CODE in "${OK_CODES[@]}"; do
+         if [ $ONLY_200_OK -eq 1 ] && [ $CODE -ne 200 ]; then
+            continue
+         fi
+      
          if [[ $CODE == $CURL_CODE ]]; then
             STATUS="OK"
@@ -1143 +1157 @@
          fi
       done
+      # Also check it against the "OK" codes besides 200 if the --only-200-ok argument was received
+      if [ $ONLY_200_OK -eq 1 ]; then
+         for CODE in "${OK_CODES[@]}"; do
+            if [ $CODE -eq 200 ]; then
+               continue
+            fi
+            if [[ $CODE == $CURL_CODE ]]; then
+               STATUS="NG"
+               let NG_LINKS+=1
+               break
+            fi
+         done
+      fi
    fi